Class: Retailer::PriceChecker

Inherits:
Object
  • Object
show all
Includes:
CatalogConstants
Defined in:
app/services/retailer/price_checker.rb

Overview

Service to check product prices and availability on retailer websites.
Uses Oxylabs Web Scraper API to fetch page content and extract data.

Examples:

Check a single catalog item

checker = Retailer::PriceChecker.new
result = checker.check(catalog_item)

Check all items for a catalog

checker = Retailer::PriceChecker.new
results = checker.check_catalog(catalog)

Constant Summary collapse

DEFAULT_GEO_LOCATIONS =

Default ZIP codes by country

{
  'USA' => '60047',  # Lake Zurich, IL (WarmlyYours HQ area)
  'CAN' => 'Canada', # country-level (Oxylabs-recommended; consistent national pricing)
  'DEU' => '10115',  # Berlin
  'GBR' => 'SW1A1AA', # London
  'FRA' => '75001',  # Paris
  'ITA' => '00100',  # Rome
  'ESP' => '28001',  # Madrid
  'NLD' => '1011',   # Amsterdam
  'POL' => '00-001', # Warsaw
  'SWE' => '11120',  # Stockholm
  'BEL' => '1000'    # Brussels
}.freeze
CATALOG_RETAILER_TYPES =

Retailer type detection based on catalog ID
Uses CatalogConstants for maintainability

{
  # Home Depot
  HOME_DEPOT_USA => :home_depot_usa,
  HOME_DEPOT_CANADA => :home_depot_canada,
  # Costco
  COSTCO_CANADA => :costco_canada,
  # Wayfair
  WAYFAIR_USA => :wayfair,
  WAYFAIR_CANADA => :wayfair,
  WAYFAIR_GERMANY => :wayfair,
  # Rona/Lowe's
  RONA_CANADA => :rona,
  LOWES_USA => :lowes,
  LOWES_CANADA => :lowes,
  # Build.com (Ferguson Home)
  BUILD_COM => :build_com,
  # Amazon Seller Marketplaces
  AMAZON_SC_US_CATALOG_ID => :amazon,
  AMAZON_SC_CA_CATALOG_ID => :amazon,
  AMAZON_SC_FR_CATALOG_ID => :amazon,
  AMAZON_SC_IT_CATALOG_ID => :amazon,
  AMAZON_SC_ES_CATALOG_ID => :amazon,
  AMAZON_SC_DE_CATALOG_ID => :amazon,
  AMAZON_SC_NL_CATALOG_ID => :amazon,
  AMAZON_SC_PL_CATALOG_ID => :amazon,
  AMAZON_SC_UK_CATALOG_ID => :amazon,
  AMAZON_SC_SE_CATALOG_ID => :amazon,
  AMAZON_SC_BE_CATALOG_ID => :amazon,
  # Walmart Seller Marketplaces
  WALMART_SELLER_USA => :walmart_usa,
  WALMART_SELLER_CANADA => :walmart_canada,
  # Canadian Tire
  CANADIAN_TIRE => :canadian_tire,
  # Best Buy Canada
  BESTBUY_CANADA => :bestbuy_canada
}.freeze

Constants included from CatalogConstants

CatalogConstants::ALL_MAIN_CATALOG_IDS, CatalogConstants::AMAZON_CATALOG_IDS, CatalogConstants::AMAZON_CA_CATALOG_IDS, CatalogConstants::AMAZON_EU_CATALOG_IDS, CatalogConstants::AMAZON_NA_SELLER_IDS, CatalogConstants::AMAZON_SC_BE_CATALOG_ID, CatalogConstants::AMAZON_SC_CATALOG_IDS, CatalogConstants::AMAZON_SC_CA_CATALOG_ID, CatalogConstants::AMAZON_SC_DE_CATALOG_ID, CatalogConstants::AMAZON_SC_ES_CATALOG_ID, CatalogConstants::AMAZON_SC_FR_CATALOG_ID, CatalogConstants::AMAZON_SC_IT_CATALOG_ID, CatalogConstants::AMAZON_SC_NL_CATALOG_ID, CatalogConstants::AMAZON_SC_PL_CATALOG_ID, CatalogConstants::AMAZON_SC_SE_CATALOG_ID, CatalogConstants::AMAZON_SC_UK_CATALOG_ID, CatalogConstants::AMAZON_SC_US_CATALOG_ID, CatalogConstants::AMAZON_SELLER_IDS, CatalogConstants::AMAZON_US_CATALOG_IDS, CatalogConstants::AMAZON_VC_CATALOG_IDS, CatalogConstants::AMAZON_VC_CA_CATALOG_ID, CatalogConstants::AMAZON_VC_CA_CATALOG_IDS, CatalogConstants::AMAZON_VC_DIRECT_FULFILLMENT_CATALOG_IDS, CatalogConstants::AMAZON_VC_US_CATALOG_IDS, CatalogConstants::AMAZON_VC_US_WASN4_CATALOG_ID, CatalogConstants::AMAZON_VC_US_WAX7V_CATALOG_ID, CatalogConstants::AMAZON_VC_WAT0F_CA_CATALOG_ID, CatalogConstants::AMAZON_VC_WAT4D_CA_CATALOG_ID, CatalogConstants::AMAZON_VENDOR_CODE_TO_CATALOG_ID, CatalogConstants::BESTBUY_CANADA, CatalogConstants::BUILD_COM, CatalogConstants::CANADIAN_TIRE, CatalogConstants::CA_CATALOG_ID, CatalogConstants::COSTCO_CANADA, CatalogConstants::COSTCO_CATALOGS, CatalogConstants::COSTCO_USA, CatalogConstants::EU_CATALOG_ID, CatalogConstants::HOME_DEPOT_CANADA, CatalogConstants::HOME_DEPOT_CATALOGS, CatalogConstants::HOME_DEPOT_USA, CatalogConstants::HOUZZ, CatalogConstants::LOCALE_TO_CATALOG, CatalogConstants::LOWES_CANADA, CatalogConstants::LOWES_USA, CatalogConstants::RONA_CANADA, CatalogConstants::US_CATALOG_ID, CatalogConstants::WALMART_CATALOGS, CatalogConstants::WALMART_SELLER_CANADA, CatalogConstants::WALMART_SELLER_USA, CatalogConstants::WAYFAIR_CANADA, CatalogConstants::WAYFAIR_CATALOGS, CatalogConstants::WAYFAIR_GERMANY, CatalogConstants::WAYFAIR_USA

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from CatalogConstants

amazon_catalog?, amazon_seller_catalog?, costco_catalog?, home_depot_catalog?, walmart_catalog?, wayfair_catalog?

Constructor Details

#initialize(options = {}) ⇒ PriceChecker

Returns a new instance of PriceChecker.

Parameters:

  • options (Hash) (defaults to: {}) —

    configurable options

Options Hash (options):

  • api (Retailer::OxylabsApi) —

    API client to use (defaults to a new instance with a 120s timeout)

  • url_constructor (Retailer::UrlConstructor) —

    URL constructor to use (defaults to a new instance)

  • logger (Logger) —

    logger to use (defaults to Rails.logger)



76
77
78
79
80
 
# File 'app/services/retailer/price_checker.rb', line 76

def initialize(options = {})
  @api = options[:api] || Retailer::OxylabsApi.new(timeout: 120)
  @url_constructor = options[:url_constructor] || Retailer::UrlConstructor.new
  @logger = options[:logger] || Rails.logger
end

Instance Attribute Details

#api ⇒ Object (readonly)

Returns the value of attribute api.



70
71
72
 
# File 'app/services/retailer/price_checker.rb', line 70

def api
  @api
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



70
71
72
 
# File 'app/services/retailer/price_checker.rb', line 70

def logger
  @logger
end

#url_constructor ⇒ Object (readonly)

Returns the value of attribute url_constructor.



70
71
72
 
# File 'app/services/retailer/price_checker.rb', line 70

def url_constructor
  @url_constructor
end

Instance Method Details

#check(catalog_item, geo_location: nil) ⇒ CatalogItemRetailerProbe

Check a single catalog item and record the result

Parameters:

  • catalog_item (CatalogItem)
  • geo_location (String) (defaults to: nil) —

    Optional ZIP code override

Returns:



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'app/services/retailer/price_checker.rb', line 86

def check(catalog_item, geo_location: nil)
  return nil unless catalog_item.probeable?

  catalog = catalog_item.catalog
  retailer_type = detect_retailer_type(catalog)

  # Costco groups all variants under one parent URL with no per-variant
  # page, so a rendered scrape only reads the page default. Use Costco's
  # JSON price/stock APIs instead — see Retailer::CostcoProbe.
  return Retailer::CostcoProbe.new.probe(catalog_item) if retailer_type.in?(%i[costco_canada costco_usa])

  # Walmart the same way, and for a stronger reason: we are the seller, so the
  # Marketplace API is the system of record. It also names WHY a listing is
  # unbuyable (UNPUBLISHED / SYSTEM_PROBLEM / 404), which a scrape cannot —
  # every one of those reads as "page had no price". See Retailer::WalmartProbe.
  return Retailer::WalmartProbe.new.probe(catalog_item) if Retailer::WalmartProbe.handles?(catalog.id)

  # Get URL - prefer hardcoded, fall back to generated
  url = url_constructor.product_url(catalog_item)
  return nil if url.blank? && retailer_type != :amazon

  geo_location ||= default_geo_location(catalog_item)
  start_time = Time.current

  # Create the check record with the URL that will be used
  check = catalog_item.retailer_probes.build(
    status: 'pending',
    geo_location: geo_location,
    url: url
  )

  begin
    result = fetch_product(catalog_item, catalog, retailer_type, geo_location)
    process_result(check, result, start_time, retailer_type, catalog_item)
  rescue StandardError => e
    check.status = 'failed'
    check.error_message = "#{e.class}: #{e.message}"
    @logger.error "[PriceChecker] Error checking #{catalog_item.id}: #{e.message}"
  end

  check.save!

  # A price came back, so whatever was wrong is over: put the item back on the
  # top rung of the probe-backoff ladder (Retailer::ProbeAutoSkipper).
  Retailer::ProbeAutoSkipper.record_success!(catalog_item) if check.status == 'success'

  # Update the catalog item's url_valid, url_last_checked, and retail_price fields
  update_attrs = {
    url_valid: check.page_accessible?,
    url_last_checked: Time.current
  }

  # Only store retail_price for vendor (1P) catalogs where the retailer
  # sets the price. For marketplace (3P) catalogs the price is ours, so
  # writing it back as "advertised price" is circular and confusing.
  if check.status == 'success' && check.price.present? && catalog.retailer_type_vendor?
    update_attrs[:retail_price] = check.price
    update_attrs[:retailer_price_updated_at] = Time.current
  end

  # Update URL if we discovered a better one (e.g., from search results)
  if @discovered_url.present? && is_direct_product_url?(@discovered_url)
    clean_url = strip_tracking_params(@discovered_url)
    if catalog_item.url.blank? || catalog_item.url.include?('search')
      update_attrs[:url] = clean_url
      @logger.info "[PriceChecker] Discovered URL for item #{catalog_item.id}: #{clean_url}"
    end
  end

  catalog_item.update_columns(update_attrs)

  # Stop re-probing items whose URL has been failing for many runs in a row.
  Retailer::ProbeAutoSkipper.maybe_skip!(catalog_item) unless check.status == 'success'

  check
end

#check_all_retailers ⇒ Hash

Check all active items across all retailer catalogs

Returns:

  • (Hash) —

    Summary of results



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'app/services/retailer/price_checker.rb', line 180

def check_all_retailers
  results = { total: 0, success: 0, failed: 0, catalogs: {} }

  retailer_catalog_items.find_each do |item|
    check_record = check(item)
    next unless check_record

    results[:total] += 1
    results[:success] += 1 if check_record.status == 'success'
    results[:failed] += 1 if check_record.status.in?(%w[failed not_found])

    catalog_name = item.catalog.name
    results[:catalogs][catalog_name] ||= { success: 0, failed: 0 }
    results[:catalogs][catalog_name][:success] += 1 if check_record.status == 'success'
    results[:catalogs][catalog_name][:failed] += 1 if check_record.status.in?(%w[failed not_found])
  end

  results
end

#check_catalog(catalog) ⇒ Array<CatalogItemRetailerProbe>

Check all active items in a catalog

Parameters:

Returns:



166
167
168
169
170
171
172
173
174
175
176
 
# File 'app/services/retailer/price_checker.rb', line 166

def check_catalog(catalog)
  items = catalog.catalog_items
                 .where(state: 'active')
                 .probeable

  # Amazon items are keyed by ASIN and Costco items by retailer item
  # number, so neither needs a stored URL; every other catalog does.
  items = items.where.not(url: [nil, '']) unless catalog.name.match?(/amazon/i) || catalog.is_costco?

  items.filter_map { |item| check(item) }
end