Class: DataImport

Inherits:
ApplicationRecord show all
Defined in:
app/models/data_import.rb

Overview

== Schema Information

Table name: data_imports
Database name: primary

id :integer not null, primary key
activity_description :string(255)
activity_notes :text
activity_schedule_offset_days :integer
complete :boolean
default_affiliation :string
default_country_iso3 :string(3)
disable_address_correction :boolean default(FALSE)
do_not_auto_assign :boolean default(FALSE)
file_name :string
file_uid :string
first_row_is_header :boolean default(TRUE), not null
initial_state :string(255)
model_version :integer default(2), not null
name :string(255)
note :text
reception_type :string(255)
state :string
created_at :datetime
updated_at :datetime
activity_assigned_resource_id :integer
activity_type_id :integer
creator_id :integer
default_buying_group_id :integer
default_catalog_id :integer
default_primary_sales_rep_id :integer
default_profile_id :integer
reuse_data_import_id :integer
source_id :integer
tier2_program_pricing_id :integer
updater_id :integer

Indexes

data_imports_reuse_data_import_id_idx (reuse_data_import_id)

Foreign Keys

fk_rails_... (reuse_data_import_id => data_imports.id) ON DELETE => nullify

Defined Under Namespace

Classes: BaseParser, CsvParser, XlsxParser

Constant Summary

Constants included from Schedulable

Schedulable::SIMPLE_FORM_OPTIONS

Instance Attribute Summary collapse

Has many collapse

Belongs to collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from ApplicationRecord

ransackable_associations, ransackable_attributes, ransackable_scopes, ransortable_attributes, #to_relation

Methods included from Schedulable

config

Methods included from Models::AfterCommittable

#after_commit

Methods included from Models::EventPublishable

#publish_event

Instance Attribute Details

#name ⇒ Object (readonly) 



67
 
# File 'app/models/data_import.rb', line 67

validates :name, uniqueness: true

Class Method Details

.options_for_select ⇒ Object 



84
85
86
 
# File 'app/models/data_import.rb', line 84

def self.options_for_select
  order(Arel.sql('COALESCE(updated_at,created_at) DESC')).limit(50).pluck(:name, :id)
end

Instance Method Details

#activity_resource ⇒ Employee

Returns:

See Also:



61
 
# File 'app/models/data_import.rb', line 61

belongs_to :activity_resource, class_name: 'Employee', foreign_key: 'activity_assigned_resource_id', optional: true

#activity_type ⇒ ActivityType

Returns:

See Also:



60
 
# File 'app/models/data_import.rb', line 60

belongs_to :activity_type, optional: true

#can_delete? ⇒ Boolean

can only delete if not processed

Returns:

  • (Boolean) 


161
162
163
 
# File 'app/models/data_import.rb', line 161

def can_delete?
  !data_import_rows.where.not(import_state: 'not_imported').exists?
end

#data_import_cells ⇒ ActiveRecord::Relation<DataImportCell>

Returns:

See Also:



52
 
# File 'app/models/data_import.rb', line 52

has_many :data_import_cells, through: :data_import_columns

#data_import_columns ⇒ ActiveRecord::Relation<DataImportColumn>

Returns:

See Also:



51
 
# File 'app/models/data_import.rb', line 51

has_many :data_import_columns, inverse_of: :data_import, dependent: :delete_all

#data_import_rows ⇒ ActiveRecord::Relation<DataImportRow>

Returns:

See Also:



53
 
# File 'app/models/data_import.rb', line 53

has_many :data_import_rows, inverse_of: :data_import, dependent: :delete_all

#deep_dup ⇒ Object 



71
72
73
74
75
76
 
# File 'app/models/data_import.rb', line 71

def deep_dup
  deep_clone(
    include: :data_import_columns,
    except: %i[file_uid file_name reuse_data_import_id state]
  )
end

#default_buying_group ⇒ BuyingGroup

Returns:

See Also:



59
 
# File 'app/models/data_import.rb', line 59

belongs_to :default_buying_group, class_name: 'BuyingGroup', optional: true

#default_catalog ⇒ Catalog

Returns:

See Also:



58
 
# File 'app/models/data_import.rb', line 58

belongs_to :default_catalog, class_name: 'Catalog', optional: true

#default_profile ⇒ Profile

Returns:

See Also:



57
 
# File 'app/models/data_import.rb', line 57

belongs_to :default_profile, class_name: 'Profile', optional: true

#notes_data_set ⇒ Object 



126
127
128
 
# File 'app/models/data_import.rb', line 126

def notes_data_set
  DataDictionarySet.find_by(destination: 'notes')
end

#parser ⇒ Object 



130
131
132
 
# File 'app/models/data_import.rb', line 130

def parser
  @parser ||= DataImport::BaseParser.instantiate(file.path, file.name, first_row_is_header: first_row_is_header)
end

#populate_cells ⇒ Object 



134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
 
# File 'app/models/data_import.rb', line 134

def populate_cells
  cells = []
  # Retrieve a map of column index to data import column id
  columns = setup_header_row(parser.column_names)
  rows_processed = 0
  total_ops = parser.size + 1
  # Transaction globbing, makes inserts faster and all in or all fail
  DataImportRow.transaction do
    parser.each_with_index do |row, row_index|
      next if row_index.zero? && first_row_is_header

      Rails.logger.info "Processing row #{row_index}"
      if (new_row = setup_value_row(row, row_index, columns))
        cells += new_row
        rows_processed += 1
      end
      yield({ position: rows_processed, total: total_ops, message: "Splitting row #{rows_processed} into cells" }) if block_given?
    end
    Rails.logger.info " ** Starting Inserts of #{cells.size} cells"
    yield({ position: total_ops, total: total_ops, message: "Inserting #{cells.size} cells in bulk (might take a while)" }) if block_given?
    DataImportCell.insert_all(cells)
    Rails.logger.info "Populate cells completed, total cells #{cells.size}, total rows: #{rows_processed}"
  end
  rows_processed
end

#process_import(row_ids = []) ⇒ Object 



165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'app/models/data_import.rb', line 165

def process_import(row_ids = [])
  res = { requested: 0, queued: 0, processed: 0, mode: nil }
  rows = data_import_rows.queued.order(:row_index)
  rows = rows.where(id: row_ids) if row_ids.present?
  res[:requested] = rows.size
  res[:mode] = res[:requested] > 10 ? :queued : :direct
  rows.find_each do |dir|
    if dir.customer
      dir.complete!
    elsif res[:mode] == :queued
      res[:queued] += 1
      DataImportWorker.perform_async(dir.id)
    elsif res[:mode] == :direct
      res[:processed] += 1
      dir.import
    end
  end
  res
end

#reset ⇒ Object 



88
89
90
91
92
93
 
# File 'app/models/data_import.rb', line 88

def reset
  Rails.logger.info ' ** Purging and deleting rows and columns'
  data_import_rows.delete_all
  data_import_columns.delete_all
  Rails.logger.info ' ** Purge complete'
end

#setup_header_row(row) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'app/models/data_import.rb', line 95

def setup_header_row(row)
  columns = {}
  notes_data_set_id = notes_data_set.id
  row.each_with_index do |column_name, index|
    dic = data_import_columns.where(column_index: index)
                             .first_or_create(name: column_name,
                                              data_dictionary_set_id: notes_data_set_id)
    Rails.logger.info " ** Mapping column #{index} #{dic.name} to data import column #{dic.id}"
    columns[index] = dic.id
  end

  columns
end

#setup_value_row(row, row_index, columns) ⇒ Object 



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'app/models/data_import.rb', line 109

def setup_value_row(row, row_index, columns)
  cells = []
  # Create our row
  dr = data_import_rows.create! row_index: row_index
  Rails.logger.info " ** Row created id #{dr.id} for row index #{row_index}"
  # Now insert each column's cell value
  row.each_with_index do |cell_value, column_index|
    next if cell_value.blank?

    cells << DataImportCell.new(data_import_column_id: columns[column_index],
                                data_import_row_id: dr.id,
                                content: cell_value)
  end

  cells
end

#tier2_program_pricing ⇒ Coupon::Tier2ProgramPricing

Returns:

See Also:



55
 
# File 'app/models/data_import.rb', line 55

belongs_to :tier2_program_pricing, class_name: 'Coupon::Tier2ProgramPricing', optional: true

#validate_file ⇒ Object 



185
186
187
188
189
 
# File 'app/models/data_import.rb', line 185

def validate_file
  return unless file_changed? && file&.stored?

  self.errors += parser.validate
end