Class EmbedderProcessor

constructor

• new EmbedderProcessor(
      provider: EmbeddingProvider,
      options?: {
          batchSize?: number;
          concurrency?: number;
          logger?: Logger;
          maxBatchChars?: number;
          minIngestionBatchSize?: number;
          onProgress?: (completed: number, total: number) => void;
          rateLimitMs?: number;
          retry?: RetryOptions;
          saveIntervalMs?: number;
          vectorStoreName?: string;
      },
  ): EmbedderProcessor

  Parameters

  • provider: EmbeddingProvider
  • options: {
        batchSize?: number;
        concurrency?: number;
        logger?: Logger;
        maxBatchChars?: number;
        minIngestionBatchSize?: number;
        onProgress?: (completed: number, total: number) => void;
        rateLimitMs?: number;
        retry?: RetryOptions;
        saveIntervalMs?: number;
        vectorStoreName?: string;
    } = {}

  Returns EmbedderProcessor

OptionalonProgress

embedBatch

• embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]>

  Parameters

  • chunks: Chunk[]

  Returns Promise<EmbeddedChunk[]>

embedChunk

• embedChunk(chunk: Chunk): Promise<EmbeddedChunk>

  Parameters

  • chunk: Chunk

  Returns Promise<EmbeddedChunk>

embedChunks

• embedChunks(chunks: Chunk[]): Promise<EmbeddedChunk[]>

  Embed a list of chunks, handling sub-batching by batchSize and maxBatchChars. Use this instead of embedBatch() when the caller doesn't know the provider's preferred batch size.

  Parameters

  • chunks: Chunk[]

  Returns Promise<EmbeddedChunk[]>

getChunksToEmbed

• getChunksToEmbed(
      db: VirageDb,
      chunks: Chunk[],
      force?: boolean,
  ): Promise<
      {
          chunksToEmbed: Chunk[];
          isNewEmbeddingSpace: boolean;
          meta: EmbeddingsMeta
          | null;
      },
  >

  Parameters

  • db: VirageDb
  • chunks: Chunk[]
  • force: boolean = false

  Returns Promise<
      {
          chunksToEmbed: Chunk[];
          isNewEmbeddingSpace: boolean;
          meta: EmbeddingsMeta
          | null;
      },
  >

run

• run(
      db: VirageDb,
      chunksFile: string,
      force?: boolean,
      onIntermediateBatch?: () => Promise<void>,
  ): Promise<EmbeddedChunk[]>

  Parameters

  • db: VirageDb
  • chunksFile: string
  • force: boolean = false
  • OptionalonIntermediateBatch: () => Promise<void>

  Returns Promise<EmbeddedChunk[]>

saveEmbeddings

• saveEmbeddings(
      db: VirageDb,
      newEmbeddings: EmbeddedChunk[],
      meta: EmbeddingsMeta,
  ): Promise<void>

  Parameters

  • db: VirageDb
  • newEmbeddings: EmbeddedChunk[]
  • meta: EmbeddingsMeta

  Returns Promise<void>