Parte 3: Configuración del ejemplo RAG y validación del pipeline de recuperación

#!pip install lancedb rerankers together -q

import pandas as pd
import os
from together import Together

os.environ["TOGETHER_API_KEY"] = ""
client = Together(api_key=os.environ.get('TOGETHER_API_KEY'))

df = pd.read_csv("./MM-Demo/final_balanced_sample_dataset.csv")

df.head()

Filename  \
0  d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg   
1  5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg   
2  b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg   
3  87846aa9-86cc-404a-af2c-7e8fe941081d.jpg   
4  04fa06fb-d71a-4293-9804-fe799375a682.jpg   

                                               Title Size  Gender  \
0  Stylish and Trendy Tank Top with Celestial Design    M       F   
1                           Classic White Sweatshirt    M       F   
2                                       Grey T-shirt    M  Unisex   
3                          Long-Sleeved V-Neck Shirt    L       U   
4                     Silver Metallic Buckle Sandals    L       F   

                                         Description Category    Type  
0  This white tank top is a stylish and trendy pi...     Tops  Casual  
1  This classic white sweatshirt is a timeless pi...     Tops  Casual  
2  This is a short-sleeved, crew neck t-shirt tha...  T-Shirt  Casual  
3  A long-sleeved, V-neck shirt with a solid purp...     Tops  Casual  
4  These silver metallic buckle sandals feature a...    Shoes  Casual

df["Description"][0]

'This white tank top is a stylish and trendy piece of clothing that features a captivating celestial design. The design showcases a mesmerizing array of stars, planets, and moons in black and white, creating a visually appealing and eye-catching pattern. The tank top has a relaxed fit with a round neckline and sleeveless design, making it a comfortable and versatile piece that can be worn on its own or layered under a cardigan or jacket. Perfect for a casual day out or a night on the town, this tank top is a must-have for any fashion-conscious woman.'

import lancedb
from lancedb.pydantic import LanceModel, Vector
from lancedb.embeddings import get_registry
from lancedb.rerankers import ColbertReranker
from pathlib import Path
from PIL import Image
import pandas as pd

model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cuda")

class Schema(LanceModel):
    Filename: str
    Title: str
    Size: str
    Gender: str
    Description: str = model.SourceField()
    Category: str
    Type: str
    vector: Vector(model.ndims()) = model.VectorField()

db = lancedb.connect("~/.lancedb")
tbl = db.create_table(name="clothes", schema=Schema, mode="overwrite")

# Function to clean and convert data types
def clean_data(df):
    # Convert all columns to string type
    for col in df.columns:
        df[col] = df[col].astype(str)
    
    # Remove any rows with NaN values
    df = df.dropna()
    
    return df

# Clean the data
cleaned_df = clean_data(df)

# Convert cleaned DataFrame to list of dictionaries
data = cleaned_df.to_dict('records')

empty_count = len([d for d in data if d.get('Description', '').strip() == ''])
print(f"Number of empty descriptions: {empty_count}")

# automatically generate vectors
tbl.add(data)

tbl.search().to_pandas()

def show_image_from_response(rs):
    names = rs["Filename"].to_list()
    return [Image.open(Path("archive") / "images_compressed" / name) for name in names]

Number of empty descriptions: 0

# automatically generate vectors
tbl.add(data)

tbl.search().to_pandas()

Filename  \
0  d7ed1d64-2c65-427f-9ae4-eb4aaa3e2389.jpg   
1  5c1b7a77-1fa3-4af8-9722-cd38e45d89da.jpg   
2  b2e084c7-e3a0-4182-8671-b908544a7cf2.jpg   
3  87846aa9-86cc-404a-af2c-7e8fe941081d.jpg   
4  04fa06fb-d71a-4293-9804-fe799375a682.jpg   
5  8f576f1a-839d-4fb2-a224-a4700b2d05da.jpg   
6  e976a8f6-6731-485f-8a9a-2872a5208818.jpg   
7  bbf0d9c7-663d-46d1-a9f8-66e8e5678541.jpg   
8  e25a7faa-7a49-4e72-a7ef-e74427f77784.jpg   
9  d995ac1f-fbd0-482c-a308-dafb6a93cfd0.jpg   

                                               Title Size  Gender  \
0  Stylish and Trendy Tank Top with Celestial Design    M       F   
1                           Classic White Sweatshirt    M       F   
2                                       Grey T-shirt    M  Unisex   
3                          Long-Sleeved V-Neck Shirt    L       U   
4                     Silver Metallic Buckle Sandals    L       F   
5                         Orange Long Sleeve T-Shirt    S       U   
6                                      Green T-Shirt    L       M   
7                     White Quarter Zip Pullover Top    L       U   
8                         Aviator Sunglasses T-Shirt    M       F   
9                                          Beige Top    M       F   

                                         Description Category    Type  \
0  This white tank top is a stylish and trendy pi...     Tops  Casual   
1  This classic white sweatshirt is a timeless pi...     Tops  Casual   
2  This is a short-sleeved, crew neck t-shirt tha...  T-Shirt  Casual   
3  A long-sleeved, V-neck shirt with a solid purp...     Tops  Casual   
4  These silver metallic buckle sandals feature a...    Shoes  Casual   
5  This long sleeve t-shirt is made of a lightwei...     Tops  Casual   
6  The green t-shirt is a relaxed fit with a V-ne...     Tops  Casual   
7  This pullover top has a quarter zip closure at...     Tops  Casual   
8  This sleeveless red T-shirt features a large b...     Tops  Casual   
9  This beige top has a boat neck, a pocket in fr...     Tops  Casual   

                                              vector  
0  [-0.06355423, 0.02385288, 0.03382446, -0.00212...  
1  [-0.011691423, 0.049270794, 0.030319242, -0.02...  
2  [-0.010150914, 0.09637642, -0.0012558334, 0.04...  
3  [-0.058710814, 0.053951632, -0.047531255, 0.03...  
4  [0.0123484805, 0.02398385, 0.059779372, -0.006...  
5  [-0.016778396, 0.06664105, -0.010597771, 0.052...  
6  [0.015946185, 0.07543863, 0.032447945, 0.00442...  
7  [-0.05592024, 0.03401515, -0.0057456787, 0.030...  
8  [-0.040058654, 0.090077035, -0.020889325, 0.03...  
9  [-0.020475557, 0.032990508, 0.0145089375, -0.0...

Enfoque 1/3: Usando búsqueda de texto

El primer enfoque que tomaremos es:

• Subir una imagen y pedirle a 3.2-11B-Instruct de Together que describa la imagen.
• Luego intentaremos encontrar imágenes similares en el conjunto de datos.

Nota: Esto es para validar que nuestro pipeline está funcionando; en la demostración final, le pediremos a Llama que encuentre ropa complementaria.

prompt = prompt + "Please answer within 100 words consisely and only provide the answer, don't" \
            "repeat any word from the input, start your response from the clothing items that can be paired with the input"
response = client.chat.completions.create(
    model="meta-llama/Llama-Vision-Free",
    messages=[
                {
                "role": "user",
                "content": [
                        {
                                "type": "text",
                                "text": prompt
                        }
                ]
        },
    ],
    max_tokens=512,
    temperature=0.7,
    top_p=0.7,
    top_k=50,
    repetition_penalty=1,
    stop=["<|eot_id|>","<|eom_id|>"],
)
print(response.choices[0].message.content)
response = response.choices[0].message.content

Jeans are a classic pairing option. Dark wash skinny jeans create a sleek look, while light wash boyfriend jeans add a casual touch. Black trousers offer a dressier alternative. Khaki chinos provide a neutral background for the plaid pattern. Corduroy pants add a rustic feel. Wool trousers bring a sophisticated element. Leather pants create a bold statement.

# automatically converted to vector representation
rs = tbl.search(response).limit(3).to_pandas()
rs

Filename               Title Size Gender  \
0  7a6e7ddb-ac03-4d78-9d71-75320e5913da.jpg  Light-Washed Jeans    S      F   
1  7a6e7ddb-ac03-4d78-9d71-75320e5913da.jpg  Light-Washed Jeans    S      F   
2  8db4d594-4688-4447-a5b9-cd5c68b1870e.jpg          Black Pant    S      U   

                                         Description Category    Type  \
0  These light-washed jeans are a pair of stylish...    Jeans  Casual   
1  These light-washed jeans are a pair of stylish...    Jeans  Casual   
2  This pair of black pants appears to be made of...    Pants  Casual   

                                              vector  _distance  
0  [-0.063473865, 0.060125574, 0.02189098, -0.002...   0.407864  
1  [-0.063473865, 0.060125574, 0.02189098, -0.002...   0.407864  
2  [-0.014388849, 0.082536675, 0.011225891, -0.00...   0.412849

imgs = show_image_from_response(rs)

imgs[0]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=400x534>

imgs[1]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=400x534>

imgs[2]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=400x533>

Enfoque 2/3: Usando búsqueda de texto completo

tbl.create_fts_index("Description", replace=True)

rs = tbl.search(response, query_type="fts").limit(5).to_pandas()
rs

filename  \
0  0e20abb0-d56a-4f83-b254-9db70fb92794.jpg   
1  10e51a41-3729-472e-b288-f3b8f895577a.jpg   
2  bd289469-607c-45df-bad3-ed1057e277bf.jpg   
3  64c5d20e-7cec-4965-8c50-315389f44b22.jpg   
4  b0c03127-9dfb-4573-8934-1958396937bf.jpg   

                                         Description  \
0  This shirt is a long-sleeved, black and grey s...   
1  This plaid shirt features a classic plaid patt...   
2  This is a red and black plaid skirt with a cla...   
3  These light washed blue jeans are distressed w...   
4  This red flannel plaid shirt has a classic and...   

                                              vector  \
0  [-0.045025624, 0.041662034, -0.0006505165, 0.0...   
1  [-0.044541422, 0.05006024, -0.04257077, 0.0401...   
2  [-0.021971317, 0.12156065, -0.002515854, 0.034...   
3  [-0.07545003, 0.05375045, 0.00353413, -0.01551...   
4  [-0.03013145, 0.060508166, -0.03642542, 0.0301...   

                                            Title Size Category    Type  \
0  Black and Grey Striped Shirt with Pink Accents    S     Tops  Casual   
1                Plaid shirt with embroidered hem    L     Tops  Casual   
2                       Red and Black Plaid Skirt    S   Skirts  Casual   
3                  Light Washed Blue Ripped Jeans    L    Jeans  Casual   
4                         Red Flannel Plaid Shirt    S   Shirts  Casual   

  Gender     _score  
0      F  52.089787  
1      F  35.616638  
2      F  34.096905  
3      M  29.116760  
4      M  28.444168

imgs = show_image_from_response(rs)

imgs[-1]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=400x533>

imgs[-2]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=400x534>

imgs[-3]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=533x400>

Enfoque 3/3: Usando búsqueda híbrida

from lancedb.rerankers import ColbertReranker

reranker = ColbertReranker(column="Description")
rs = tbl.search(response, query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
rs

Loading ColBERTRanker model colbert-ir/colbertv2.0
No device set
Using device cuda
No dtype set
Using dtype torch.float16
Loading model colbert-ir/colbertv2.0, this might take a while...
Linear Dim set to: {linear_dim} for downcasting

filename  \
0  0e20abb0-d56a-4f83-b254-9db70fb92794.jpg   
1  10e51a41-3729-472e-b288-f3b8f895577a.jpg   
2  300a4125-acec-46aa-b0b9-68b4b2142c70.jpg   
3  bd289469-607c-45df-bad3-ed1057e277bf.jpg   
4  b0c03127-9dfb-4573-8934-1958396937bf.jpg   

                                         Description  \
0  This shirt is a long-sleeved, black and grey s...   
1  This plaid shirt features a classic plaid patt...   
2  These plaid shorts are a stylish and comfortab...   
3  This is a red and black plaid skirt with a cla...   
4  This red flannel plaid shirt has a classic and...   

                                              vector  \
0  [-0.045025624, 0.041662034, -0.0006505165, 0.0...   
1  [-0.044541422, 0.05006024, -0.04257077, 0.0401...   
2  [-0.002765015, 0.030705247, -0.010424565, -0.0...   
3  [-0.021971317, 0.12156065, -0.002515854, 0.034...   
4  [-0.03013145, 0.060508166, -0.03642542, 0.0301...   

                                            Title Size Category    Type  \
0  Black and Grey Striped Shirt with Pink Accents    S     Tops  Casual   
1                Plaid shirt with embroidered hem    L     Tops  Casual   
2                                    Plaid Shorts    S    Pants  Casual   
3                       Red and Black Plaid Skirt    S   Skirts  Casual   
4                         Red Flannel Plaid Shirt    S   Shirts  Casual   

  Gender  _relevance_score  
0      F          0.744141  
1      F          0.735352  
2      M          0.728516  
3      F          0.699219  
4      M          0.679199

imgs = show_image_from_response(rs)

imgs[-1]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=400x533>

imgs[-2]

<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=533x400>

#fin