Dev vectors bucket

alibabacloud_oss_v2/vectors/endpoints.py

@@ -6,9 +6,9 @@
 def from_region(region: str, etype: str) -> str:
     """Generate vectors endpoint from region"""
     if etype == "internal":
-        return f"oss-{region}-internal.oss-vectors.aliyuncs.com"
+        return f"{region}-internal.oss-vectors.aliyuncs.com"
     else:
-        return f"oss-{region}.oss-vectors.aliyuncs.com"
+        return f"{region}.oss-vectors.aliyuncs.com"
 
 
 class VectorsEndpointProvider(EndpointProvider):

alibabacloud_oss_v2/vectors/models/index_basic.py

@@ -89,24 +89,20 @@ class GetVectorIndexResult(serde.ResultModel):
 
     _attribute_map = {
         'index': {'tag': 'output', 'position': 'body', 'rename': 'index', 'type': 'dict'},
-        'vector_bucket_name': {'tag': 'output', 'position': 'body', 'rename': 'vectorBucketName', 'type': 'str'},
     }
 
 
     def __init__(
         self,
         index: Optional[Dict] = None,
-        vector_bucket_name: Optional[str] = None,
         **kwargs: Any
     ) -> None:
         """
         Args:
             index (Dict, optional): The vector index information.
-            vector_bucket_name (str, optional): The name of the vector bucket.
         """
         super().__init__(**kwargs)
         self.index = index
-        self.vector_bucket_name = vector_bucket_name
 
 
 # List

sample/vectors/get_vector_index.py

@@ -36,7 +36,6 @@ def main():
 
     if result.index:
         print(f'index name: {result.index}')
-        print(f'vector bucket name: {result.vector_bucket_name}')
 
 
 

tests/integration/__init__.py

@@ -18,6 +18,7 @@
 ACCESS_ID = os.getenv("OSS_TEST_ACCESS_KEY_ID")
 ACCESS_KEY = os.getenv("OSS_TEST_ACCESS_KEY_SECRET")
 ENDPOINT = os.getenv("OSS_TEST_ENDPOINT")
+VECTOR_ENDPOINT = os.getenv("OSS_TEST_VECTOR_ENDPOINT")
 REGION = os.getenv("OSS_TEST_REGION", "cn-hangzhou")
 RAM_ROLE_ARN = os.getenv("OSS_TEST_RAM_ROLE_ARN")
 SIGNATURE_VERSION = os.getenv("OSS_TEST_SIGNATURE_VERSION")
@@ -102,7 +103,7 @@ def get_vectors_client() -> oss_vectors.Client:
     cfg = oss.config.load_default()
     cfg.credentials_provider = oss.credentials.StaticCredentialsProvider(ACCESS_ID, ACCESS_KEY)
     cfg.region = REGION
-    cfg.endpoint = ENDPOINT
+    cfg.endpoint = VECTOR_ENDPOINT
     cfg.account_id = USER_ID
     return oss_vectors.Client(cfg)
 
@@ -147,6 +148,9 @@ def random_str(n):
 def random_bucket_name():
     return BUCKETNAME_PREFIX + random_lowstr(4) + '-' + str(int(datetime.datetime.now(datetime.timezone.utc).timestamp()))
 
+def random_short_bucket_name():
+    return BUCKETNAME_PREFIX + random_lowstr(7)
+
 def clean_objects(client:oss.Client, bucket_name:str) -> None:
     marker = ''
     is_truncated = True
@@ -253,7 +257,80 @@ def sts_assume_role(access_key_id:str, access_key_secret:str, role_arn:str) -> d
     response = requests.get(assume_url)
 
     return json.loads(response.content)
-
+
+
+def clean_vector_buckets(prefix: str) -> None:
+    vector_client = get_vectors_client()
+
+    paginator = vector_client.list_vector_buckets_paginator()
+    for page in paginator.iter_page(oss_vectors.models.ListVectorBucketsRequest(prefix=prefix)):
+        for bucket in page.buckets:
+            actual_bucket_name = bucket.name.split(':')[-1]
+
+            # Clean all content in the bucket (indexes and vectors)
+            clean_vector_bucket_content(vector_client, actual_bucket_name)
+
+            # Delete the bucket itself
+            delete_vector_bucket(actual_bucket_name)
+
+def clean_vector_bucket_content(client: oss_vectors.Client, full_bucket_name: str) -> None:
+    """
+    Clean all content in the specified vector bucket (including indexes and vectors)
+    """
+    # Clean all vector indexes
+    clean_vector_indexes(client, full_bucket_name)
+
+
+def clean_vector_indexes(client: oss_vectors.Client, bucket_name: str) -> None:
+    """
+    Clean all vector indexes in the specified bucket and their contained vectors
+    """
+    paginator_index = client.list_vector_indexes_paginator()
+    for page_index in paginator_index.iter_page(
+            oss_vectors.models.ListVectorIndexesRequest(bucket=bucket_name)
+    ):
+        for index in page_index.indexes:
+            # Clean all vectors in the index
+            clean_vectors(client, bucket_name, index.get("indexName"))
+
+            # Delete the vector index
+            client.delete_vector_index(oss_vectors.models.DeleteVectorIndexRequest(
+                bucket=bucket_name,
+                index_name=index.get("indexName"),
+            ))
+
+
+def clean_vectors(client: oss_vectors.Client, bucket_name: str, index_name: str) -> None:
+    """
+    Clean all vectors in the specified index
+    """
+    paginator = client.list_vectors_paginator()
+    request = oss_vectors.models.ListVectorsRequest(
+        bucket=bucket_name,
+        index_name=index_name,
+    )
+
+    for page_vector in paginator.iter_page(request):
+        keys = []
+        for vec in page_vector.vectors:
+            keys.append(vec.get("key"))
+
+        # Delete all vectors on the current page
+        if keys:
+            client.delete_vectors(oss_vectors.models.DeleteVectorsRequest(
+                bucket=bucket_name,
+                index_name=index_name,
+                keys=keys
+            ))
+
+
+def delete_vector_bucket(bucket_name: str) -> None:
+    """
+    Delete vector bucket
+    """
+    client = get_vectors_client()
+    client.delete_vector_bucket(oss_vectors.models.DeleteVectorBucketRequest(bucket=bucket_name))
+
 
 class TestIntegrationVectors(TestIntegration):
 
@@ -262,8 +339,16 @@ def setUpClass(cls):
         TestIntegration.setUpClass()
         cls.vector_client = get_vectors_client()
 
+        vector_bucket_name = random_short_bucket_name()
+        result = cls.vector_client.put_vector_bucket(
+            oss_vectors.models.PutVectorBucketRequest(
+                bucket=vector_bucket_name,
+            )
+        )
+        cls.vector_bucket_name = vector_bucket_name
+
     @classmethod
     def tearDownClass(cls):
         TestIntegration.tearDownClass()
-
+        clean_vector_buckets(BUCKETNAME_PREFIX)
 

tests/integration/vector/test_vector_basic_client.py

@@ -1,7 +1,8 @@
 # pylint: skip-file
 
+from ast import literal_eval
 import alibabacloud_oss_v2.vectors as oss_vectors
-from tests.integration import TestIntegrationVectors, random_bucket_name
+from tests.integration import TestIntegrationVectors, random_short_bucket_name
 
 
 class TestVectorBasic(TestIntegrationVectors):
@@ -10,7 +11,9 @@ class TestVectorBasic(TestIntegrationVectors):
     def test_vector_basic(self):
         """Test put, get, list, delete and query vector operations."""
         # 1. Create bucket
-        bucket_name = random_bucket_name()
+        bucket_name = random_short_bucket_name()
+        vector_key = "vector-key-1"
+
         result = self.vector_client.put_vector_bucket(oss_vectors.models.PutVectorBucketRequest(
             bucket=bucket_name,
         ))
@@ -20,33 +23,32 @@ def test_vector_basic(self):
         self.assertEqual(24, len(result.headers.get('x-oss-request-id')))
 
         # 2. Create index (required for vector operations)
-        index_name = 'test-index'
-        dimension = 128
-        distance_metric = 'EUCLIDEAN'
-        data_type = 'vector'
+        index_name = 'testIndexForIntegrationVector'
+        dimension = 3
+        distance_metric = 'cosine'
+        data_type = 'float32'
         metadata = {"nonFilterableMetadataKeys": ["key1", "key2"]}
-        put_index_result = self.vector_client.put_vector_index(oss_vectors.models.PutVectorIndexRequest(
+
+        put_index_request = oss_vectors.models.PutVectorIndexRequest(
             bucket=bucket_name,
             data_type=data_type,
             dimension=dimension,
             distance_metric=distance_metric,
             index_name=index_name,
             metadata=metadata
-        ))
-        self.assertEqual(200, put_index_result.status_code)
-        self.assertEqual('OK', put_index_result.status)
-        self.assertEqual(24, len(put_index_result.request_id))
-        self.assertEqual(24, len(put_index_result.headers.get('x-oss-request-id')))
-        self.assertIsNotNone(put_index_result.index)
-        self.assertEqual(index_name, put_index_result.index.index_name)
-        self.assertEqual(dimension, put_index_result.index.dimension)
-        self.assertEqual(distance_metric, put_index_result.index.distance_metric)
+        )
+
+        put_result = self.vector_client.put_vector_index(put_index_request)
+        self.assertEqual(200, put_result.status_code)
+        self.assertEqual('OK', put_result.status)
+        self.assertEqual(24, len(put_result.request_id))
+        self.assertEqual(24, len(put_result.headers.get('x-oss-request-id')))
 
         # 3. Put vectors
         vectors_to_put = [
             {
                 "data": {"float32": [0.1, 0.2, 0.3]},
-                "key": "vector-key-1",
+                "key": vector_key,
                 "metadata": {"key1": "value1", "key2": "value2"}
             }
         ]
@@ -60,75 +62,83 @@ def test_vector_basic(self):
         self.assertEqual('OK', put_result.status)
         self.assertEqual(24, len(put_result.request_id))
         self.assertEqual(24, len(put_result.headers.get('x-oss-request-id')))
-        self.assertIsNotNone(put_result.result)
 
         # 4. Get vectors
         get_result = self.vector_client.get_vectors(oss_vectors.models.GetVectorsRequest(
             bucket=bucket_name,
             index_name=index_name,
-            keys=['vector-key-1'],
+            keys=[vector_key],
             return_data=True,
             return_metadata=True
         ))
         self.assertEqual(200, get_result.status_code)
         self.assertEqual('OK', get_result.status)
         self.assertEqual(24, len(get_result.request_id))
         self.assertEqual(24, len(get_result.headers.get('x-oss-request-id')))
-        self.assertIsNotNone(get_result.result)
-        self.assertIsNotNone(get_result.result.vectors)
-        self.assertEqual(1, len(get_result.result.vectors))
-        self.assertEqual("vector-key-1", get_result.result.vectors[0].key)
-        self.assertIsNotNone(get_result.result.vectors[0].data)
-        self.assertIsNotNone(get_result.result.vectors[0].metadata)
+        self.assertIsNotNone(get_result.vectors)
+        self.assertEqual(1, len(get_result.vectors))
+        self.assertEqual(vector_key, get_result.vectors[0].get('key'))
+        self.assertIsNotNone(get_result.vectors[0].get('data'))
+        self.assertIsNotNone(get_result.vectors[0].get('metadata'))
+        self.assertEqual(literal_eval('{"key1": "value1", "key2": "value2"}'), get_result.vectors[0].get('metadata'))
 
         # 5. List vectors
         list_result = self.vector_client.list_vectors(oss_vectors.models.ListVectorsRequest(
             bucket=bucket_name,
             index_name=index_name,
             max_results=100,
-            next_token='',
             return_data=True,
             return_metadata=True,
-            segment_count=5,
-            segment_index=2
+            segment_count=2,
+            segment_index=1
         ))
         self.assertEqual(200, list_result.status_code)
         self.assertEqual('OK', list_result.status)
         self.assertEqual(24, len(list_result.request_id))
         self.assertEqual(24, len(list_result.headers.get('x-oss-request-id')))
-        self.assertIsNotNone(list_result.result)
-        self.assertIsNotNone(list_result.result.vectors)
+        self.assertIsNotNone(list_result.vectors)
         # Check that we have at least one vector in the list
-        self.assertGreaterEqual(len(list_result.result.vectors), 1)
+        self.assertGreaterEqual(len(list_result.vectors), 0)
+        self.assertEqual(vector_key, list_result.vectors[0].get('key'))
+        self.assertIsNotNone(list_result.vectors[0].get('data'))
+        self.assertIsNotNone(list_result.vectors[0].get('metadata'))
+        self.assertEqual(literal_eval('{"key1": "value1", "key2": "value2"}'), list_result.vectors[0].get('metadata'))
+
 
         # 6. Query vectors
-        query_vector = [0.1, 0.2, 0.3]
+        query_filter = {
+            "$and": [{
+                "type": {
+                    "$nin": ["comedy", "documentary"]
+                }
+            }]
+        }
+        query_vector = {"float32": [0.1, 0.2, 0.3]}
         query_result = self.vector_client.query_vectors(oss_vectors.models.QueryVectorsRequest(
             bucket=bucket_name,
             index_name=index_name,
-            vector=query_vector,
-            top_k=1
+            query_vector=query_vector,
+            filter=query_filter,
+            return_distance=True,
+            return_metadata=True,
+            top_k=10
         ))
         self.assertEqual(200, query_result.status_code)
         self.assertEqual('OK', query_result.status)
         self.assertEqual(24, len(query_result.request_id))
         self.assertEqual(24, len(query_result.headers.get('x-oss-request-id')))
-        self.assertIsNotNone(query_result.result)
-        self.assertIsNotNone(query_result.result.matches)
-        # Check that we have matches returned
-        self.assertGreaterEqual(len(query_result.result.matches), 0)
+        self.assertIsNotNone(query_result)
+
 
         # 7. Delete vectors
         delete_result = self.vector_client.delete_vectors(oss_vectors.models.DeleteVectorsRequest(
             bucket=bucket_name,
             index_name=index_name,
-            keys=['vector-key-1']
+            keys=[vector_key]
         ))
-        self.assertEqual(200, delete_result.status_code)
-        self.assertEqual('OK', delete_result.status)
+        self.assertEqual(204, delete_result.status_code)
         self.assertEqual(24, len(delete_result.request_id))
         self.assertEqual(24, len(delete_result.headers.get('x-oss-request-id')))
-        self.assertIsNotNone(delete_result.result)
 
         # 8. Get index to verify it exists
         get_index_result = self.vector_client.get_vector_index(oss_vectors.models.GetVectorIndexRequest(
@@ -140,7 +150,7 @@ def test_vector_basic(self):
         self.assertEqual(24, len(get_index_result.request_id))
         self.assertEqual(24, len(get_index_result.headers.get('x-oss-request-id')))
         self.assertIsNotNone(get_index_result.index)
-        self.assertEqual(index_name, get_index_result.index.index_name)
+        self.assertEqual(index_name, get_index_result.index.get('indexName'))
 
         # 9. List indexes
         list_index_result = self.vector_client.list_vector_indexes(oss_vectors.models.ListVectorIndexesRequest(
@@ -152,17 +162,17 @@ def test_vector_basic(self):
         self.assertEqual(24, len(list_index_result.headers.get('x-oss-request-id')))
         self.assertIsNotNone(list_index_result.indexes)
         # Check that we have at least one index in the list
-        self.assertGreaterEqual(len(list_index_result.indexes), 1)
+        self.assertEqual(len(list_index_result.indexes), 1)
 
         # Verify our index is in the list
         found_index = None
         for index in list_index_result.indexes:
-            if index.index_name == index_name:
+            if index.get('indexName') == index_name:
                 found_index = index
                 break
         self.assertIsNotNone(found_index)
-        self.assertEqual(index_name, found_index.index_name)
-        self.assertEqual(dimension, found_index.dimension)
+        self.assertEqual(index_name, found_index.get('indexName'))
+        self.assertEqual(dimension, found_index.get('dimension'))
 
         # 10. Delete index (cleanup)
         delete_index_result = self.vector_client.delete_vector_index(oss_vectors.models.DeleteVectorIndexRequest(