Class: EmbeddingWorker

Inherits:
Object
  • Object
show all
Includes:
Sidekiq::Job
Defined in:
app/workers/embedding_worker.rb

Overview

Background worker for generating content embeddings.
Uses RubyLLM to generate vector embeddings for semantic search.

Examples:

Queue embedding generation

EmbeddingWorker.perform_async('Showcase', 123)

Queue with specific content type

EmbeddingWorker.perform_async('Video', 456, content_types: ['transcript'])

Instance Method Summary collapse

Instance Method Details

#perform(class_name, id, options = {}) ⇒ Object

Parameters:

  • class_name (String) —

    The model class name (e.g., 'Showcase', 'Post')

  • id (Integer) —

    The record ID

  • options (Hash) (defaults to: {}) —

    Optional parameters

Options Hash (options):

  • :content_types (Array<String>) —

    Specific content types to embed

  • :force (Boolean) —

    Force regeneration even if not stale

  • :chunked (Boolean) —

    Use chunked embedding for large content (e.g., PDFs)

  • :locales (Array<String>) —

    Override locales to embed (default: record.embeddable_locales)



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'app/workers/embedding_worker.rb', line 44

def perform(class_name, id, options = {})
  record = find_record(class_name, id)
  return unless record
  return unless retain_embedding?(record)

  # Symbol reads below are safe whichever way callers spell their keys:
  # Sidekiq::NormalizeArgsMiddleware::Server hands #perform a
  # HashWithIndifferentAccess. See lib/sidekiq/normalize_args_middleware.rb.
  content_types = options[:content_types]&.map(&:to_sym) || record.class.embeddable_content_types
  force   = options[:force].to_b
  # Records that opt into chunking (e.g. publications, whose merged docling +
  # vision content routinely exceeds the ~8k-token window) always take the
  # chunked path regardless of the caller's flag — so every enqueue site
  # (auto after_commit, vision worker, extraction worker) stays consistent.
  # Mixing a single `primary` row with `primary_chunk_*` rows for one record
  # would leave stale/duplicate vectors in retrieval.
  chunked = options[:chunked].to_b || record.try(:embeddable_chunked?)

  # Models that define embeddable_locales (e.g. Post → ['en-US', 'en-CA']) will have
  # a separate embedding generated for each locale. All other models fall back to a
  # single locale via locale_for_embedding.
  locales = options[:locales]&.map(&:to_s).presence || record.embeddable_locales

  log_info "Generating embeddings for #{class_name}##{id} " \
           "(types: #{content_types.join(', ')}, locales: #{locales.join(', ')}, chunked: #{chunked})"

  results = content_types.flat_map do |content_type|
    locales.map do |locale|
      if chunked && record.respond_to?(:generate_chunked_embeddings!)
        generate_chunked_with_rate_limit(record, content_type, force, locale)
      else
        generate_with_rate_limit(record, content_type, force, locale)
      end
    end
  end.flatten

  successful = results.count { |r| r.is_a?(ContentEmbedding) }
  log_info "Completed #{class_name}##{id}: #{successful} embeddings generated"
rescue ActiveRecord::RecordNotFound
  log_info "Record not found: #{class_name}##{id} - skipping"
rescue Embedding::Gemini::PermanentError => e
  Rails.event.notify(
    'warning.embedding.permanent_failure',
    embeddable_type: class_name,
    embeddable_id: id,
    exception_class: e.class.name,
    exception_message: e.message
  )
rescue Embedding::Gemini::Error => e
  log_error "Gemini error for #{class_name}##{id}: #{e.message}"
  raise # Re-raise to trigger Sidekiq retry
rescue RubyLLM::Error => e
  log_error "RubyLLM error for #{class_name}##{id}: #{e.message}"
  raise # Re-raise to trigger retry
rescue StandardError => e
  log_error "Unexpected error for #{class_name}##{id}: #{e.message}"
  ErrorReporting.error(e)
  raise
end