OpenAI Embeddings学习笔记

poetry new --src openai-embedding-sample
cd openai-embedding-sample

openai-embedding-sample
├── pyproject.toml
├── README.md
├── src
│   └── openai_embedding_sample
│       └── __init__.py
└── tests
    └── __init__.py

poetry config virtualenvs.in-project true --local
poetry env use python3.11

poetry source add --priority=primary private-pypi https://mirrors.aliyun.com/pypi/simple/

poetry add openai pandas matplotlib plotly scikit-learn numpy tiktoken

poetry add --group=tests pytest

import pandas as pd
import tiktoken


def main():
    # 加载数据集
    input_datapath = "data/fine_food_reviews_1k.csv"
    df = pd.read_csv(input_datapath)
    df = df[["Time", "ProductId", "UserId", "Score", "Summary", "Text"]]
    df = df.dropna()

    df["combined"] = (
        "Title: " + df.Summary.str.strip() + "; Content: " + df.Text.str.strip()
    )
    print(df.head(2))

    embedding_model = "text-embedding-3-small"
    embedding_encoding = "cl100k_base"
    max_tokens = 8000  # the maximum for text-embedding-3-small is 8191

    embedding_model = "bge-large-en-v1.5"
    max_tokens = 512  # the maximum for bge-large-en-v1.5 is 512

    # subsample to 1k most recent reviews and remove samples that are too long
    top_n = 1000
    df = df.sort_values("Time").tail(
        top_n * 2
    )  # first cut to first 2k entries, assuming less than half will be filtered out
    df.drop("Time", axis=1, inplace=True)

    encoding = tiktoken.get_encoding(embedding_encoding)

    # omit reviews that are too long to embed
    df["n_tokens"] = df.combined.apply(lambda x: len(encoding.encode(x)))
    df = df[df.n_tokens <= max_tokens].tail(top_n)
    print(len(df))
    print(df.head(2))

from openai import OpenAI
import os


def test_openai_api_embeddings():
    OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE")
    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
    embedding_model = "bge-large-en-v1.5"
    client = OpenAI(base_url=OPENAI_API_BASE, api_key=OPENAI_API_KEY)
    res = client.embeddings.create(input="abc", model=embedding_model)
    print(res.data[0].embedding)

pytest -rP tests/test_openai.py::test_openai_api_embeddings

from typing import List
from openai import OpenAI
import os

OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE")
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")

client = OpenAI(base_url=OPENAI_API_BASE, api_key=OPENAI_API_KEY, max_retries=5)


def get_embedding(text: str, model="text-embedding-3-small", **kwargs) -> List[float]:
    # replace newlines, which can negatively affect performance.
    text = text.replace("\n", " ")

    response = client.embeddings.create(input=[text], model=model, **kwargs)

    return response.data[0].embedding

    # Ensure you have your API key set in your environment per the README: https://github.com/openai/openai-python#usage
    # This may take a few minutes
    df["embedding"] = df.combined.apply(
        lambda x: get_embedding(x, model="bge-large-en-v1.5")
    )
    df.to_csv("data/fine_food_reviews_with_embeddings_1k.csv")

    # 读取生成嵌入的文件
    df = pd.read_csv("data/fine_food_reviews_with_embeddings_1k.csv", index_col=0)
    # 查看embeddings结果和长度
    print(df["embedding"])

python -m openai_embedding_sample

12     [-0.017417438328266144, -0.018280575051903725,...
13     [-0.027951912954449654, -0.018266700208187103,...
14     [0.027293438091874123, -0.02840059995651245, -...
15     [0.004309102427214384, -0.021994683891534805, ...
16     [0.017969520762562752, -0.0392923504114151, -0...
                             ...                        
447    [0.007145790383219719, -0.0035732914693653584,...
436    [0.013964557088911533, -0.01780335046350956, 0...
437    [-0.032477207481861115, 0.0280800461769104, -0...
438    [-0.014685509726405144, 0.0015338457887992263,...
439    [-0.009211455471813679, -0.04697975143790245, ...

    # 查看embeddings结果第一行
    print(df["embedding"][0])
    print("length:", len(df["embedding"][0]))
    print("type:", type(df["embedding"][0]))

[0.0267749335616827, 0.003123185131698847, -0.024137750267982483, 0.057977546006441116, -0.053905557841062546, -0.04009933024644852, 0.015665242448449135, 0.06601973623037338, 0.03214592859148979, 0.02665616199374199, 0.019556688144803047, 0.024528663605451584, 0.025058161467313766, -0.009058943018317223, -0.04273372143507004, -0.03410712629556656, -0.0060263825580477715, -0.037118904292583466, -0.
......
0009543468477204442, -0.0009659237111918628, -0.05392676964402199, -0.011576066724956036, -0.007504361681640148, -0.0032890550792217255, -0.03377753123641014, -0.03155573084950447, 0.05228942632675171, -0.012785147875547409, -0.02052222564816475, -0.06280946731567383, -0.026425695046782494, 0.01089983806014061, 0.006608058698475361, 0.051717400550842285, -0.022326190024614334, -0.01687932014465332, -0.03479322791099548, 0.01124760415405035, 0.04013797640800476, 0.02007046528160572, -0.023960063233971596, 0.05589267984032631, -0.006869079079478979]
length: 22696
type: <class 'str'>

    # 将字符串转换为向量
    df["embedding"] = df["embedding"].apply(literal_eval).to_list()
    print("length:", len(df["embedding"][0]))
    print("type:", type(df["embedding"][0]))

length: 1024
type: <class 'list'>

    print(df.head(2))

     ProductId         UserId  Score                     Summary                                               Text                                           combined  n_tokens                                          embedding
12  B000K8T3OQ  AK43Y4WT6FFR3      1  Broken in a million pieces  Chips were broken into small pieces. Problem i...  Title: Broken in a million pieces; Content: Ch...       120  [-0.017417438328266144, -0.018280575051903725,...
13  B0051C0J6M  AWFA8N9IXELVH      1       Deceptive description  On Oct 9 I ordered from a different vendor the...  Title: Deceptive description; Content: On Oct ...       157  [-0.027951912954449654, -0.018266700208187103,...

    # 将df["embedding"]转换为numpy数组
    matrix = np.array(df["embedding"].to_list())
    assert len(matrix) == 997  # 数据行数
    print(type(matrix))  # <class 'numpy.ndarray'>
    assert len(matrix[0]) == 1024  # 向量维度
    print(type(matrix[0]))  # <class 'numpy.ndarray'>

    from sklearn.manifold import TSNE

    # Create a t-SNE model and transform the data
    tsne = TSNE(
        n_components=2, perplexity=15, random_state=42, init="random", learning_rate=200
    )
    vis_dims = tsne.fit_transform(matrix)
    assert vis_dims.shape == (997, 2)

    import matplotlib.pyplot as plt
    import matplotlib
    import numpy as np

    colors = ["red", "darkorange", "gold", "turquoise", "darkgreen"]
    x = [x for x, y in vis_dims]
    y = [y for x, y in vis_dims]
    color_indices = df.Score.values - 1

    colormap = matplotlib.colors.ListedColormap(colors)
    plt.scatter(x, y, c=color_indices, cmap=colormap, alpha=0.3)
    for score in [0, 1, 2, 3, 4]:
        avg_x = np.array(x)[df.Score - 1 == score].mean()
        avg_y = np.array(y)[df.Score - 1 == score].mean()
        color = colors[score]
        plt.scatter(avg_x, avg_y, marker="x", color=color, s=100)

    plt.title("Amazon ratings visualized in language using t-SNE")
    plt.savefig("visualized-using-t-sne.png")

import numpy as np
import pandas as pd
from ast import literal_eval


def test_k_means_clustering_in_python_using_openai():
    # load data
    datafile_path = "./data/fine_food_reviews_with_embeddings_1k.csv"

    df = pd.read_csv(datafile_path)
    df["embedding"] = df.embedding.apply(literal_eval).apply(np.array)
    # convert string to numpy array
    matrix = np.vstack(df.embedding.values)
    assert matrix.shape == (997, 1024)

pytest -rP tests/test_openai.py::test_k_means_clustering_in_python_using_openai

    from sklearn.cluster import KMeans

    n_clusters = 4

    kmeans = KMeans(n_clusters=n_clusters, init="k-means++", random_state=42)
    kmeans.fit(matrix)
    labels = kmeans.labels_
    df["Cluster"] = labels

    print(df.groupby("Cluster").Score.mean().sort_values())

Cluster
3    3.706186
2    4.144444
1    4.325658
0    4.467085
Name: Score, dtype: float64

    from sklearn.manifold import TSNE
    import matplotlib
    import matplotlib.pyplot as plt

    tsne = TSNE(
        n_components=2, perplexity=15, random_state=42, init="random", learning_rate=200
    )
    vis_dims2 = tsne.fit_transform(matrix)

    x = [x for x, y in vis_dims2]
    y = [y for x, y in vis_dims2]

    for category, color in enumerate(["purple", "green", "red", "blue"]):
        xs = np.array(x)[df.Cluster == category]
        ys = np.array(y)[df.Cluster == category]
        plt.scatter(xs, ys, color=color, alpha=0.3)

        avg_x = xs.mean()
        avg_y = ys.mean()

        plt.scatter(avg_x, avg_y, marker="x", color=color, s=100)
    plt.title("Clusters identified visualized in language 2d using t-SNE")
    plt.savefig("clusters-visualized-using-t-sne.png")

    from openai import OpenAI
    import os

    OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE")
    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")

    client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_API_BASE)

    # Reading a review which belong to each group.
    rev_per_cluster = 5

    for i in range(n_clusters):
        print(f"Cluster {i} Theme:", end=" ")

        reviews = "\n".join(
            df[df.Cluster == i]
            .combined.str.replace("Title: ", "")
            .str.replace("\n\nContent: ", ":  ")
            .sample(rev_per_cluster, random_state=42)
            .values
        )

        messages = [
            {
                "role": "user",
                "content": f'What do the following customer reviews have in common?\n\nCustomer reviews:\n"""\n{reviews}\n"""\n\nTheme:',
            }
        ]

        response = client.chat.completions.create(
            model="qwen2-7b-instruct",
            messages=messages,
            temperature=0,
            max_tokens=64,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0,
        )
        print(response.choices[0].message.content.replace("\n", ""))

        sample_cluster_rows = df[df.Cluster == i].sample(
            rev_per_cluster, random_state=42
        )
        for j in range(rev_per_cluster):
            print(sample_cluster_rows.Score.values[j], end=", ")
            print(sample_cluster_rows.Summary.values[j], end=":   ")
            print(sample_cluster_rows.Text.str[:70].values[j])

        print("-" * 100)

Cluster
3    3.706186
2    4.144444
1    4.325658
0    4.467085
Name: Score, dtype: float64
Cluster 0 Theme: The customer reviews share a common theme of praising specific products for their health benefits, taste, and suitability for particular dietary needs, while also expressing dissatisfaction with pricing. Here's a breakdown:1. **Nutritional Value and Health Benefits**: The reviews highlight the positive aspects of the products in terms of nutrition, such as the
5, Yum!:   I loved the V8 Pomegranate Blueberry energy drink! It was very good an
5, Michael Seasons Unsalted Potato Chips:   These are the best chips ever!  They are unsalted and being on a low s
5, Licorice for the Gluten Free:   It is a very good tasting alternative to other licorice. Being Gluten 
5, Michael Seasons Unsalted Potato Chips:   These are the best chips ever!  They are unsalted and being on a low s
3, This price is ridiculous!:   There's nothing wrong with variety, except when it comes at too high a
----------------------------------------------------------------------------------------------------
Cluster 1 Theme: The common theme among these customer reviews is that they all express satisfaction with the product, despite some minor issues or preferences. Each review highlights a positive aspect of the product, such as flavor, taste, or overall satisfaction, while also mentioning a slight drawback or a desire for improvement. The reviewers seem to be repeat customers who
4, Like this tea:   This tea has a nice flavor although I wish it was a little stronger.  
5, Love this stuff!!:   I Have been ordering this k cup cappucino drink for awhile now and hav
5, Addictive:   I never cared much for the original Cherry Coke, but I love the new Ch
4, Like this tea:   This tea has a nice flavor although I wish it was a little stronger.  
2, twice the price of my expensive grocery store!:   I know what I am getting as far as product, taste, etc., love the PRIM
----------------------------------------------------------------------------------------------------
Cluster 2 Theme: The common theme among these customer reviews is the evaluation and comparison of various coffee and tea products, particularly those designed for Keurig single-serve coffee makers. The reviewers discuss the quality, taste, value, and convenience of the products, often contrasting them with other options on the market, such as traditional K-Cups
5, Exactly what I was looking for: Fast, fantastic Chai!:   I was skeptical as to how good an all-in-one Chai Tea for Keurig could
5, THE Alternative To High Priced K-Cups You've Been LOOKING For!:   You love K-Cups, but get annoyed with paying $1.50+ apiece? This produ
5, Delish!!:   So yummy... Drinking it Black coffee or w cream this coffee is delish 
5, Good coffee.:   This is the best Donut Shop Blend out there.  Though most k-cups taste
1, way too bitter:   I love coffee. I am used to strong coffee, I have lots of it. I drink 
----------------------------------------------------------------------------------------------------
Cluster 3 Theme: The common theme among these customer reviews is the discrepancy between the product description and the actual product received, leading to dissatisfaction or disappointment among the reviewers. Each review highlights a different product and issue, but the core concern revolves around misrepresentation or misunderstanding of what the product entails. This theme is evident in the following aspects:1
5, Favorite chew toy!:   This is the second one of these antlers that I've ordered (the first o
2, Contains aspartame!:   I ordered this gum based on the premise that it contained xylitol rath
5, won't kid you:   I won't kid you there is nothing better than a real hot dog.  I love h
5, Dogs love it.:   This is the "all gone" treat after dinner.  It's the only treat that t
1, Deceptive description:   On Oct 9 I ordered from a different vendor the same product 1.2 oz- 6 
----------------------------------------------------------------------------------------------------

def test_semantic_text_search_using_embeddings():
    import pandas as pd
    import numpy as np
    from ast import literal_eval

    datafile_path = "data/fine_food_reviews_with_embeddings_1k.csv"

    df = pd.read_csv(datafile_path)
    df["embedding"] = df.embedding.apply(literal_eval).apply(np.array)

import numpy as np


def cosine_similarity(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

from openai_embedding_sample import get_embedding, cosine_similarity


# search through the reviews for a specific product
def search_reviews(df, product_description, n=3, pprint=True):
    product_embedding = get_embedding(product_description, model="bge-large-en-v1.5")
    df["similarity"] = df.embedding.apply(
        lambda x: cosine_similarity(x, product_embedding)
    )

    results = (
        df.sort_values("similarity", ascending=False)
        .head(n)
        .combined.str.replace("Title: ", "")
        .str.replace("; Content:", ": ")
    )
    if pprint:
        for r in results:
            print(r[:200])
            print()
    return results

def test_semantic_text_search_using_embeddings():
    import pandas as pd
    import numpy as np
    from ast import literal_eval

    datafile_path = "data/fine_food_reviews_with_embeddings_1k.csv"

    df = pd.read_csv(datafile_path)
    df["embedding"] = df.embedding.apply(literal_eval).apply(np.array)

    results = search_reviews(df, "delicious beans", n=3)
    print(results)

pytest -rP tests/test_openai.py::test_semantic_text_search_using_embeddings 


Delicious!:  I enjoy this white beans seasoning, it gives a rich flavor to the beans I just love it, my mother in law didn't know about this Zatarain's brand and now she is traying different seasoning

Delicious:  While there may be better coffee beans available, this is my first purchase and my first time grinding my own beans.  I read several reviews before purchasing this brand, and am extremely 

Good Buy:  I liked the beans. They were vacuum sealed, plump and moist. Would recommend them for any use. I personally split and stuck them in some vodka to make vanilla extract. Yum!

381    Delicious!:  I enjoy this white beans seasonin...
159    Delicious:  While there may be better coffee b...
442    Good Buy:  I liked the beans. They were vacuum...
Name: combined, dtype: object