Module: OmniAI::Tools::Browser::Elements::NearbyElementDetector

Defined in:
lib/omniai/tools/browser/elements/nearby_element_detector.rb

Overview

Handles detection of interactive elements near text matches

Class Method Summary collapse

Class Method Details

.add_nearby_interactive_elements(elements) ⇒ Object 



11
12
13
14
15
16
17
18
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 11

def add_nearby_interactive_elements(elements)
  nearby_elements = []
  elements.each do |element|
    nearby_elements += find_interactive_in_context(element)
  end
  combined = elements.to_a + nearby_elements
  combined.uniq
end

.check_parent_containers(element) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 75

def check_parent_containers(element)
  interactive_elements = []
  parent_container = element.parent
  3.times do
    break unless parent_container

    interactive_elements += find_interactive_in_container(parent_container)
    parent_container = parent_container.parent
  end
  interactive_elements
end

.check_table_context(element) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 27

def check_table_context(element)
  elements = []
  table_cell = element.ancestors("td").first || element.ancestors("th").first
  elements += find_interactive_in_container(table_cell) if table_cell
  table_row = element.ancestors("tr").first
  elements += find_interactive_in_container(table_row) if table_row
  if table_cell && table_row
    table = element.ancestors("table").first
    elements += find_interactive_in_same_column(table, table_cell, table_row) if table
  end
  elements
end

.find_column_cell_elements(row, column_index) ⇒ Object 



53
54
55
56
57
58
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 53

def find_column_cell_elements(row, column_index)
  target_cell = row.css("td, th")[column_index]
  return [] unless target_cell

  find_interactive_in_container(target_cell) + find_interactive_in_nested_tables(target_cell, column_index)
end

.find_interactive_in_container(container) ⇒ Object 



87
88
89
90
91
92
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 87

def find_interactive_in_container(container)
  return [] unless container

  container.css("input, textarea, select, button, [role='button'], [tabindex='0']")
    .reject { |el| non_interactive_element?(el) }
end

.find_interactive_in_context(element) ⇒ Object 



20
21
22
23
24
25
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 20

def find_interactive_in_context(element)
  interactive_elements = []
  interactive_elements += check_table_context(element)
  interactive_elements += check_parent_containers(element)
  interactive_elements.uniq
end

.find_interactive_in_nested_tables(cell, original_column_index) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 60

def find_interactive_in_nested_tables(cell, original_column_index)
  interactive_elements = []

  cell.css("table").each do |nested_table|
    nested_table.css("tr").each do |nested_row|
      nested_cells = nested_row.css("td, th")
      if nested_cells[original_column_index]
        interactive_elements += find_interactive_in_container(nested_cells[original_column_index])
      end
    end
  end

  interactive_elements
end

.find_interactive_in_same_column(table, header_cell, header_row) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 40

def find_interactive_in_same_column(table, header_cell, header_row)
  return [] unless table

  column_index = header_row.css("th, td").index(header_cell)
  return [] unless column_index

  interactive_elements = []
  table.css("tbody tr").each do |row|
    interactive_elements += find_column_cell_elements(row, column_index)
  end
  interactive_elements
end

.non_interactive_element?(element) ⇒ Boolean

Returns:

  • (Boolean) 


94
95
96
97
98
99
100
101
102
103
 
# File 'lib/omniai/tools/browser/elements/nearby_element_detector.rb', line 94

def non_interactive_element?(element)
  return true if element["type"] == "hidden"
  return true if element["tabindex"] == "-1"
  return true if element["aria-hidden"] == "true"

  style = element["style"]
  return true if style&.include?("display: none") || style&.include?("visibility: hidden")

  false
end