OpenAI Embeddings学习笔记

 1import pandas as pd
 2import tiktoken
 3
 4
 5def main():
 6    # 加载数据集
 7    input_datapath = "data/fine_food_reviews_1k.csv"
 8    df = pd.read_csv(input_datapath)
 9    df = df[["Time", "ProductId", "UserId", "Score", "Summary", "Text"]]
10    df = df.dropna()

1    df["combined"] = (
2        "Title: " + df.Summary.str.strip() + "; Content: " + df.Text.str.strip()
3    )
4    print(df.head(2))

1    embedding_model = "text-embedding-3-small"
2    embedding_encoding = "cl100k_base"
3    max_tokens = 8000  # the maximum for text-embedding-3-small is 8191
4
5    embedding_model = "bge-large-en-v1.5"
6    max_tokens = 512  # the maximum for bge-large-en-v1.5 is 512

 1    # subsample to 1k most recent reviews and remove samples that are too long
 2    top_n = 1000
 3    df = df.sort_values("Time").tail(
 4        top_n * 2
 5    )  # first cut to first 2k entries, assuming less than half will be filtered out
 6    df.drop("Time", axis=1, inplace=True)
 7
 8    encoding = tiktoken.get_encoding(embedding_encoding)
 9
10    # omit reviews that are too long to embed
11    df["n_tokens"] = df.combined.apply(lambda x: len(encoding.encode(x)))
12    df = df[df.n_tokens <= max_tokens].tail(top_n)
13    print(len(df))
14    print(df.head(2))

 1from openai import OpenAI
 2import os
 3
 4
 5def test_openai_api_embeddings():
 6    OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE")
 7    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
 8    embedding_model = "bge-large-en-v1.5"
 9    client = OpenAI(base_url=OPENAI_API_BASE, api_key=OPENAI_API_KEY)
10    res = client.embeddings.create(input="abc", model=embedding_model)
11    print(res.data[0].embedding)

 1from typing import List
 2from openai import OpenAI
 3import os
 4
 5OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE")
 6OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
 7
 8client = OpenAI(base_url=OPENAI_API_BASE, api_key=OPENAI_API_KEY, max_retries=5)
 9
10
11def get_embedding(text: str, model="text-embedding-3-small", **kwargs) -> List[float]:
12    # replace newlines, which can negatively affect performance.
13    text = text.replace("\n", " ")
14
15    response = client.embeddings.create(input=[text], model=model, **kwargs)
16
17    return response.data[0].embedding

1    # Ensure you have your API key set in your environment per the README: https://github.com/openai/openai-python#usage
2    # This may take a few minutes
3    df["embedding"] = df.combined.apply(
4        lambda x: get_embedding(x, model="bge-large-en-v1.5")
5    )
6    df.to_csv("data/fine_food_reviews_with_embeddings_1k.csv")

1    # 读取生成嵌入的文件
2    df = pd.read_csv("data/fine_food_reviews_with_embeddings_1k.csv", index_col=0)
3    # 查看embeddings结果和长度
4    print(df["embedding"])

1    # 查看embeddings结果第一行
2    print(df["embedding"][0])
3    print("length:", len(df["embedding"][0]))
4    print("type:", type(df["embedding"][0]))

1    # 将字符串转换为向量
2    df["embedding"] = df["embedding"].apply(literal_eval).to_list()
3    print("length:", len(df["embedding"][0]))
4    print("type:", type(df["embedding"][0]))

1    print(df.head(2))

1    # 将df["embedding"]转换为numpy数组
2    matrix = np.array(df["embedding"].to_list())
3    assert len(matrix) == 997  # 数据行数
4    print(type(matrix))  # <class 'numpy.ndarray'>
5    assert len(matrix[0]) == 1024  # 向量维度
6    print(type(matrix[0]))  # <class 'numpy.ndarray'>

1    from sklearn.manifold import TSNE
2
3    # Create a t-SNE model and transform the data
4    tsne = TSNE(
5        n_components=2, perplexity=15, random_state=42, init="random", learning_rate=200
6    )
7    vis_dims = tsne.fit_transform(matrix)
8    assert vis_dims.shape == (997, 2)

 1    import matplotlib.pyplot as plt
 2    import matplotlib
 3    import numpy as np
 4
 5    colors = ["red", "darkorange", "gold", "turquoise", "darkgreen"]
 6    x = [x for x, y in vis_dims]
 7    y = [y for x, y in vis_dims]
 8    color_indices = df.Score.values - 1
 9
10    colormap = matplotlib.colors.ListedColormap(colors)
11    plt.scatter(x, y, c=color_indices, cmap=colormap, alpha=0.3)
12    for score in [0, 1, 2, 3, 4]:
13        avg_x = np.array(x)[df.Score - 1 == score].mean()
14        avg_y = np.array(y)[df.Score - 1 == score].mean()
15        color = colors[score]
16        plt.scatter(avg_x, avg_y, marker="x", color=color, s=100)
17
18    plt.title("Amazon ratings visualized in language using t-SNE")
19    plt.savefig("visualized-using-t-sne.png")

 1import numpy as np
 2import pandas as pd
 3from ast import literal_eval
 4
 5
 6def test_k_means_clustering_in_python_using_openai():
 7    # load data
 8    datafile_path = "./data/fine_food_reviews_with_embeddings_1k.csv"
 9
10    df = pd.read_csv(datafile_path)
11    df["embedding"] = df.embedding.apply(literal_eval).apply(np.array)
12    # convert string to numpy array
13    matrix = np.vstack(df.embedding.values)
14    assert matrix.shape == (997, 1024)

 1    from sklearn.cluster import KMeans
 2
 3    n_clusters = 4
 4
 5    kmeans = KMeans(n_clusters=n_clusters, init="k-means++", random_state=42)
 6    kmeans.fit(matrix)
 7    labels = kmeans.labels_
 8    df["Cluster"] = labels
 9
10    print(df.groupby("Cluster").Score.mean().sort_values())

 1    from sklearn.manifold import TSNE
 2    import matplotlib
 3    import matplotlib.pyplot as plt
 4
 5    tsne = TSNE(
 6        n_components=2, perplexity=15, random_state=42, init="random", learning_rate=200
 7    )
 8    vis_dims2 = tsne.fit_transform(matrix)
 9
10    x = [x for x, y in vis_dims2]
11    y = [y for x, y in vis_dims2]
12
13    for category, color in enumerate(["purple", "green", "red", "blue"]):
14        xs = np.array(x)[df.Cluster == category]
15        ys = np.array(y)[df.Cluster == category]
16        plt.scatter(xs, ys, color=color, alpha=0.3)
17
18        avg_x = xs.mean()
19        avg_y = ys.mean()
20
21        plt.scatter(avg_x, avg_y, marker="x", color=color, s=100)
22    plt.title("Clusters identified visualized in language 2d using t-SNE")
23    plt.savefig("clusters-visualized-using-t-sne.png")

 1    from openai import OpenAI
 2    import os
 3
 4    OPENAI_API_BASE = os.environ.get("OPENAI_API_BASE")
 5    OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
 6
 7    client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_API_BASE)
 8
 9    # Reading a review which belong to each group.
10    rev_per_cluster = 5
11
12    for i in range(n_clusters):
13        print(f"Cluster {i} Theme:", end=" ")
14
15        reviews = "\n".join(
16            df[df.Cluster == i]
17            .combined.str.replace("Title: ", "")
18            .str.replace("\n\nContent: ", ":  ")
19            .sample(rev_per_cluster, random_state=42)
20            .values
21        )
22
23        messages = [
24            {
25                "role": "user",
26                "content": f'What do the following customer reviews have in common?\n\nCustomer reviews:\n"""\n{reviews}\n"""\n\nTheme:',
27            }
28        ]
29
30        response = client.chat.completions.create(
31            model="qwen2-7b-instruct",
32            messages=messages,
33            temperature=0,
34            max_tokens=64,
35            top_p=1,
36            frequency_penalty=0,
37            presence_penalty=0,
38        )
39        print(response.choices[0].message.content.replace("\n", ""))
40
41        sample_cluster_rows = df[df.Cluster == i].sample(
42            rev_per_cluster, random_state=42
43        )
44        for j in range(rev_per_cluster):
45            print(sample_cluster_rows.Score.values[j], end=", ")
46            print(sample_cluster_rows.Summary.values[j], end=":   ")
47            print(sample_cluster_rows.Text.str[:70].values[j])
48
49        print("-" * 100)

1def test_semantic_text_search_using_embeddings():
2    import pandas as pd
3    import numpy as np
4    from ast import literal_eval
5
6    datafile_path = "data/fine_food_reviews_with_embeddings_1k.csv"
7
8    df = pd.read_csv(datafile_path)
9    df["embedding"] = df.embedding.apply(literal_eval).apply(np.array)

1import numpy as np
2
3
4def cosine_similarity(a, b):
5    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

 1from openai_embedding_sample import get_embedding, cosine_similarity
 2
 3
 4# search through the reviews for a specific product
 5def search_reviews(df, product_description, n=3, pprint=True):
 6    product_embedding = get_embedding(product_description, model="bge-large-en-v1.5")
 7    df["similarity"] = df.embedding.apply(
 8        lambda x: cosine_similarity(x, product_embedding)
 9    )
10
11    results = (
12        df.sort_values("similarity", ascending=False)
13        .head(n)
14        .combined.str.replace("Title: ", "")
15        .str.replace("; Content:", ": ")
16    )
17    if pprint:
18        for r in results:
19            print(r[:200])
20            print()
21    return results

 1def test_semantic_text_search_using_embeddings():
 2    import pandas as pd
 3    import numpy as np
 4    from ast import literal_eval
 5
 6    datafile_path = "data/fine_food_reviews_with_embeddings_1k.csv"
 7
 8    df = pd.read_csv(datafile_path)
 9    df["embedding"] = df.embedding.apply(literal_eval).apply(np.array)
10
11    results = search_reviews(df, "delicious beans", n=3)
12    print(results)