char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Georgi Gerganovllama : fix FA when KV cache is not used (i.e. embeddings) (#12825)a19b5cef1

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
969 B35 linesraw
1import asyncio
2import asyncio.threads
3import requests
4import numpy as np
5
6
7n = 8
8
9result = []
10
11async def requests_post_async(*args, **kwargs):
12    return await asyncio.threads.to_thread(requests.post, *args, **kwargs)
13
14async def main():
15    model_url = "http://127.0.0.1:6900"
16    responses: list[requests.Response] = await asyncio.gather(*[requests_post_async(
17        url= f"{model_url}/embedding",
18        json= {"content": "a "*1022}
19    ) for i in range(n)])
20
21    for response in responses:
22        embedding = response.json()["embedding"]
23        print(embedding[-8:])
24        result.append(embedding)
25
26asyncio.run(main())
27
28# compute cosine similarity
29
30for i in range(n-1):
31    for j in range(i+1, n):
32        embedding1 = np.array(result[i])
33        embedding2 = np.array(result[j])
34        similarity = np.dot(embedding1, embedding2) / (np.linalg.norm(embedding1) * np.linalg.norm(embedding2))
35        print(f"Similarity between {i} and {j}: {similarity:.2f}")