AntflyAntfly

Search your image collection using text descriptions or find visually similar images using CLIP embeddings.

Prerequisites#

  • Antfly running with Antfly inference and ONNX Runtime (CLIP requires ONNX)
  • CLIP model: antfly inference pull openai/clip-vit-base-patch32

Step 1: Create the Table#

Create a table with a CLIP embeddings index. The template combines the image URL and caption for multimodal embedding:

	// Step 2: Create the table with embeddings index
	fmt.Println("Creating table 'images' with CLIP embeddings index...")

	// Build the embedder config (union type)
	var embedderConfig oapi.EmbedderConfig
	embedderConfig.Provider = oapi.EmbedderProviderAntfly
	embedderConfig.FromAntflyEmbedderConfig(oapi.AntflyEmbedderConfig{
		Model: "openai/clip-vit-base-patch32",
	})

	// Build the path-identified index request.
	indexConfig, err := antfly.NewCreateIndexRequest(oapi.EmbeddingsIndexConfig{
		Dimension: 512,
		Template:  "{{media url=image_url}}{{caption}}",
		Embedder:  embedderConfig,
	})
	if err != nil {
		log.Fatal(err)
	}

	err = client.CreateTable(ctx, "images", antfly.CreateTableRequest{
		Indexes: map[string]antfly.CreateIndexRequest{
			"embeddings": *indexConfig,
		},
	})
	if err != nil {
		errStr := err.Error()
		if strings.Contains(errStr, "already exists") {
			fmt.Println("Table 'images' already exists, continuing...")
		} else if strings.Contains(errStr, "model not found") {
			fmt.Printf("Warning: Embedder model not available (%v)\n", err)
			fmt.Println("Creating table without embeddings index...")
			err = client.CreateTable(ctx, "images", antfly.CreateTableRequest{})
			if err != nil {
				if strings.Contains(err.Error(), "already exists") {
					fmt.Println("Table 'images' already exists, continuing...")
				} else {
					log.Fatalf("Failed to create table: %v", err)
				}
			} else {
				fmt.Println("Created table 'images' (without embeddings)")
			}
		} else {
			log.Fatalf("Failed to create table: %v", err)
		}
	} else {
		fmt.Println("Created table 'images'")
	}

Note: ONNX Runtime is experimental. If you encounter issues like "model not found" errors, empty results, or embeddings not being computed, try restarting Antfly. Check antfly.log for errors if problems persist.

Step 2: Add a Sample Image#

Let's add the famous Utah teapot:

	// Step 3: Add a sample image (Utah teapot)
	fmt.Println("\nAdding Utah teapot sample image...")
	_, err = client.Batch(ctx, "images", antfly.BatchRequest{
		Inserts: map[string]any{
			"utah_teapot": map[string]any{
				"caption":   "Utah teapot",
				"image_url": "https://upload.wikimedia.org/wikipedia/commons/e/e7/Utah_teapot_simple_2.png",
			},
		},
	})
	if err != nil {
		log.Printf("Warning: Failed to add teapot: %v", err)
	} else {
		fmt.Println("Added Utah teapot")
	}

Antfly fetches and embeds the image automatically when using a URL.

Step 3: Search with Text#

	// Step 4: Search with text
	fmt.Println("\nSearching for '3D model teapot'...")
	results, err := client.Query(ctx, antfly.QueryRequest{
		Table:          "images",
		SemanticSearch: "3D model teapot",
		Indexes:        []string{"embeddings"},
		Limit:          5,
	})
	if err != nil {
		log.Fatalf("Query failed: %v", err)
	}

	fmt.Println("\nSearch results:")
	for _, resp := range results.Responses {
		for _, hit := range resp.Hits.Hits {
			fmt.Printf("  Score: %.4f, ID: %s\n", hit.Score, hit.ID)
		}
	}

The Utah teapot should appear as the top result:

Score: 0.0164, ID: utah_teapot
Score: 0.0161, ID: mmir_3bc4b3613ed9
Score: 0.0159, ID: mmir_83ca037bd2ad
...

Batch Import with Timing#

For larger datasets, here's how to import images in bulk. This example uses the MMIR dataset from Google Research:

func batchImport(ctx context.Context, client *antfly.AntflyClient) {
	numImages := 100
	fmt.Printf("Importing first %d images...\n", numImages)
	startTime := time.Now()
	successCount := 0

	f, err := os.Open("mmir_dataset.tsv.gz")
	if err != nil {
		log.Printf("Failed to open dataset: %v", err)
		return
	}
	defer f.Close()

	gz, err := gzip.NewReader(f)
	if err != nil {
		log.Printf("Failed to create gzip reader: %v", err)
		return
	}
	defer gz.Close()

	reader := csv.NewReader(gz)
	reader.Comma = '\t'
	reader.FieldsPerRecord = -1
	reader.Read() // Skip header

	for successCount < numImages {
		row, err := reader.Read()
		if err != nil {
			break
		}
		if len(row) < 5 {
			continue
		}

		imageURL, caption := row[0], row[4]
		// Handle b'...' format in dataset
		if strings.HasPrefix(imageURL, "b'") && strings.HasSuffix(imageURL, "'") {
			imageURL = imageURL[2 : len(imageURL)-1]
		}
		if strings.HasPrefix(caption, "b'") && strings.HasSuffix(caption, "'") {
			caption = caption[2 : len(caption)-1]
		}

		if !strings.HasPrefix(imageURL, "http") {
			continue
		}

		hash := md5.Sum([]byte(imageURL))
		docID := fmt.Sprintf("%x", hash[:6])

		_, err = client.Batch(ctx, "images", antfly.BatchRequest{
			Inserts: map[string]any{
				"mmir_" + docID: map[string]any{"caption": caption, "image_url": imageURL},
			},
		})
		if err == nil {
			successCount++
			fmt.Printf("\rImported: %d / %d", successCount, numImages)
		}
	}

	elapsed := time.Since(startTime).Seconds()
	fmt.Printf("\nImported %d images in %.1fs (%.1f images/sec)\n", successCount, elapsed, float64(successCount)/elapsed)
}

Example output:

Imported: 100 / 100
Imported 100 images in 45.2s (2.2 images/sec)

Running the Example#

# From the repository root
go run ./examples/image-search

# Or build and run
go build -o examples/image-search/image-search ./examples/image-search
./examples/image-search/image-search

To run the batch import, first download the MMIR dataset:

curl -o mmir_dataset.tsv.gz "https://storage.googleapis.com/gresearch/wit-retrieval/mmir_dataset_train-00000-of-00005.tsv.gz"

Tips#

Use visual descriptions: CLIP responds better to concrete visual concepts ("red sports car", "snowy mountain") than brand names or abstract terms.

Captions affect results: The template combines image + caption. For pure visual search, use "template": "{{media url=image_url}}" instead.