Selenium Loop Table Missing First Row And Last Column

Find the data you need here

We provide programming data of 20 most popular languages, hope to help you!

Previous PostNext Post

Selenium loop, table missing first row and last column

But the result is missing the first row (2021-11) and the last column (净投放(亿)). What have I done wrong? from selenium import webdriver driver = Selenium loop, table missing first row and last column. Ask Question Asked 7 months ago. Have updated the code. Scroll down to footer and then try to extract the details – pmadhu. Nov

from selenium import webdriver

driver = webdriver.Firefox(executable_path = '/usr/local/bin/geckodriver')
driver.get('http://www.chinamoney.com.cn/chinese/hb/')

rws = driver.find_elements_by_xpath("//table/tbody/tr")
r = len(rws)

cols = driver.find_elements_by_xpath("//thead/tr/td")
c = len(cols)

element = []
row = []
for i in range(1,r):
    for j in range(1,c):
        d=driver.find_element_by_xpath("//tr["+str(i)+"]/td["+str(j)+"]").text
        row.append(d)

element.append(row)
driver.close()

element
driver.get("http://www.chinamoney.com.cn/chinese/hb/")

# Scroll Down to footer so that the all the data in the table is accessible to selenium

driver.execute_script("arguments[0].scrollIntoView(true);",driver.find_element_by_id("main-page-foot"))

# Collect the table rows using - xpath "//table//tr", in the variable rows
rows = driver.find_elements_by_xpath("//table//tr") # rows is a list of webelements - rows of the table(tr tags of the table).
element = []

# Iterate over the list

for row in rows:
    # Get the data inside a row. Columns is also a list of webelements - "td" tags of that particular "tr"

    columns = row.find_elements_by_xpath(".//td") # Put a dot in the xpath to get an element from an element. 

    list_row = [] # Temporary list to store values of td tags
    for col in columns: 
        list_row.append(col.text) # Append all the td tags values 
    element.append(list_row) # Append the list of td tag values
print(element)
[['日期', '投放量(亿)', '回笼量(亿)', '净投放(亿)'], ['2021-11', '2200', '12200', '-10000'], ['2021-10', '13900', '12300', '1600'], ['2021-09', '11800', '5900', '5900'], ['2021-08', '4200', '2600', '1600'], ['2021-07', '2600', '3200', '-600'], ['2021-06', '3100', '2100', '1000'], ['2021-05', '1900', '2000', '-100'], ['2021-04', '2200', '2100', '100'], ['2021-03', '2400', '2500', '-100'], ['2021-02', '8300', '11440', '-3140'], ['2021-01', '10740', '12450', '-1710'], ['2020-12', '9150', '8900', '250'], ['2020-11', '17550', '17200', '350'], ['', '', '', ''], ['', '', '', '']]
from selenium import webdriver

driver = webdriver.Firefox(executable_path = '/usr/local/bin/geckodriver')
driver.get('http://www.chinamoney.com.cn/chinese/hb/')

time.sleep(5)
rws = driver.find_elements_by_xpath("//table/tbody/tr")
r = len(rws)

cols = driver.find_elements_by_xpath("//thead/tr/td")
c = len(cols)

element = []
row = []
for i in range(r):
    for j in range(c):
        d=driver.find_element_by_xpath("//tr["+str(i)+"]/td["+str(j)+"]").text
        row.append(d)

element.append(row)
driver.close()

element

Got only first row in table when using Selenium scraping (Python)

And I managed to scrape it but it resulted in only the first row of the table instead of 20 rows. I saw previous similar questions that were answered and I have tried the solutions given but my selenium was unable to locate the element when I use .// for my xpath.

for bod in driver.find_elements_by_xpath("//*[@id='proxylisttable']/tbody"):
    col = bod.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr")
    for c in col:
        ip = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[1]')
        port = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[2]')
        code = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[3]')
        country = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[4][@class = "hm"]')
        anonymity = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[5]')
        google = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[6][@class = "hm"]')
col = bod.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr")
col = bod.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr[insert count here]")
table = driver.find_element_by_xpath("//*[@id='proxylisttable']/tbody")
rows = table.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr")

for i in range (1, len(rows)+1):
    row = table.find_element_by_xpath("//*[@id='proxylisttable']/tbody/tr[" +str(i) +']')
    for c in row:
        ip = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[1]')
        port = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[2]')
        code = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[3]')
        country = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[4][@class = "hm"]')
        anonymity = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[5]')
        google = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[6][@class = "hm"]')
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

driver=webdriver.Chrome("path of the chrome driver")
driver.get('https://free-proxy-list.net/')

rows= WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@id='proxylisttable']/tbody//tr")))
for row in rows:
    ip=row.find_element_by_xpath('./td[1]').text
    port=row.find_element_by_xpath('./td[2]').text
    code=row.find_element_by_xpath('./td[3]').text
    country=row.find_element_by_xpath('./td[4]').get_attribute('textContent')
    Anonymity=row.find_element_by_xpath('./td[5]').text
    google=row.find_element_by_xpath('./td[6]').get_attribute('textContent')
    https=row.find_element_by_xpath('./td[7]').text
    lastchecked=row.find_element_by_xpath('./td[8]').get_attribute('textContent')
    print("IP :{}, Port:{}, code:{}, country:{}, Anonymity:{}, google:{}, https:{}, last_checked:{}".format(ip,port,code,country,Anonymity,google,https,lastchecked))
IP :185.132.133.173, Port:8080, code:NL, country:Netherlands, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :181.112.225.78, Port:58948, code:EC, country:Ecuador, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :134.249.149.219, Port:35795, code:UA, country:Ukraine, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :195.20.30.54, Port:55182, code:UA, country:Ukraine, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :14.102.69.170, Port:53347, code:IN, country:India, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :182.53.193.108, Port:54543, code:TH, country:Thailand, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :159.224.221.175, Port:58299, code:UA, country:Ukraine, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :36.89.188.123, Port:49725, code:ID, country:Indonesia, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :103.231.163.58, Port:43620, code:BD, country:Bangladesh, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :114.130.92.14, Port:49167, code:BD, country:Bangladesh, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :177.54.200.10, Port:49501, code:BR, country:Brazil, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :77.38.21.239, Port:8080, code:SI, country:Slovenia, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :78.137.89.161, Port:8080, code:YE, country:Yemen, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :103.216.147.49, Port:8080, code:IN, country:India, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :195.250.188.210, Port:8080, code:EE, country:Estonia, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :5.196.255.171, Port:3128, code:FR, country:France, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :109.234.112.250, Port:46675, code:GE, country:Georgia, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :186.225.48.178, Port:8080, code:BR, country:Brazil, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :101.255.64.142, Port:35401, code:ID, country:Indonesia, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :160.119.129.42, Port:57557, code:GN, country:Guinea, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
ip = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr[index]/td[1]')

How to fetch last row of a webtable in selenium

The problem is that the webtable is dynamic so I don't know the row number.I have tried below code but the row number is static there whic Stack Overflow. About; In first Line we get the Last Row count from the table. On second line we get the Last row Element from the table. Selenium cannot handle large table row count. 0.

List<WebElement> rows=driver.findElements(By.xpath("//table[1]/tbody/tr[5]"));
WebElement lastRow =  driver.findElement(By.xpath("(//table[1]/tbody/tr)[last()]"));
(locator)[position() = last()]
public class LastRowFetching {
    public static void main(String[] args) throws ParseException {
        WebDriver driver;
        System.setProperty("webdriver.chrome.driver","G://chromedriver.exe");
        driver= new ChromeDriver();
        driver.get("http://money.rediff.com/gainers/bsc/dailygroupa?");         
        //No.of rows 
        List  rows = driver.findElements(By.xpath(".//*[@id='leftcontainer']/table/tbody/tr/td[1]")); 
        System.out.println("No of rows are : " + rows.size());
        int lastRow = rows.size();

        WebElement lastRowFetch =driver.findElement(By.xpath(".//*[@id='leftcontainer']/table/tbody/tr/td["+lastRowcount+"])"));
        driver.close();
    }
}
    raise exception_class(message, screen, stacktrace)
StaleElementReferenceException: Message: Element is no longer valid
col_name = row.find_elements(By.TAG_NAME, "td")[1] # This is the Name column
def get_variables_col_values(self):
        try:
            table_id = self.driver.find_element(By.ID, 'data_configuration_variables_ct_fields_body1')
            #time.sleep(10)
            rows = table_id.find_elements(By.TAG_NAME, "tr")
            print "Rows length"
            print len(rows)
            for row in rows:
                # Get the columns
                print "cols length"
                print len(row.find_elements(By.TAG_NAME, "td"))
                col_name = row.find_elements(By.TAG_NAME, "td")[1] # This is the Name column                print "col_name.text = "
                print col_name.text   
        except NoSuchElementException, e:
           return False
WebDriverWait(self.driver, 10).until(lambda d: d.execute_script('return document.readyState') == 'complete')
Rows length
16
cols length
6
col_name.text = 
Name
cols length
6
col_name.text = 
Address
cols length
6
col_name.text = 
DOB
...
from selenium.webdriver.support import expected_conditions as EC

    def get_variables_col_values(self):
            try:
                WebDriverWait(self.driver, 10).until(EC.presence_of_all_elements_located((By.TAG_NAME,'td')))
                table_id = self.driver.find_element(By.ID, 'data_configuration_variables_ct_fields_body1')
                #time.sleep(10)
                rows = table_id.find_elements(By.TAG_NAME, "tr")
                print "Rows length"
                print len(rows)
                for row in rows:
                    # Get the columns
                    print "cols length"
                    print len(row.find_elements(By.TAG_NAME, "td"))
                    col_name = row.find_elements(By.TAG_NAME, "td")[1] # This is the Name column                print "col_name.text = "
                    print col_name.text   
            except NoSuchElementException, e:
               return False

Iterating Through Table Rows in Selenium (Python)

Iterating Through Table Rows in Selenium (Python) Ask Question Asked 6 years, My function always seems to go with the first table with that name, when I'm looking for the next one (which is the one below it). there was one missing round bracket at the end; also removed the [1] index, to match the first XML example.

<table class="datadisplaytable">
<tbody>
<tr>
<td class="dddefault">16759</td>
<td class="dddefault">MATH</td>
<td class="dddefault">123</td>
<td class="dddefault">001</td>
<td class="dddefault">Calculus</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
<tr>
<td class="dddefault">16449</td>
<td class="dddefault">PHY</td>
<td class="dddefault">456</td>
<td class="dddefault">002</td>
<td class="dddefault">Physics</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
</tbody>
</table>
def check_grades(self):
    table = []
    for i in self.driver.find_element_by_class_name("dddefault"):
        table.append(i)
    print(table)
mytable = find_element_by_css_selector('table.datadisplaytable')
for row in mytable.find_elements_by_css_selector('tr'):
    for cell in row.find_elements_by_tag_name('td'):
        print(cell.text)
h  = """<table class="datadisplaytable">
<tr>
<td class="dddefault">16759</td>
<td class="dddefault">MATH</td>
<td class="dddefault">123</td>
<td class="dddefault">001</td>
<td class="dddefault">Calculus</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
<tr>
<td class="dddefault">16449</td>
<td class="dddefault">PHY</td>
<td class="dddefault">456</td>
<td class="dddefault">002</td>
<td class="dddefault">Physics</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
</table>"""

from lxml import html
xml = html.fromstring(h)
# gets the table
table =  xml.xpath("//table[@class='datadisplaytable']")[0]


# iterate over all the rows   
for row in table.xpath(".//tr"):
     # get the text from all the td's from each row
    print([td.text for td in row.xpath(".//td[@class='dddefault'][text()])
['16759', 'MATH', '123', '001', 'Calculus']
['16449', 'PHY', '456', '002', 'Physics']
table =  driver.find_element_by_xpath("//table[@class='datadisplaytable']")

for row in table.find_elements_by_xpath(".//tr"):
    print([td.text for td in row.find_elements_by_xpath(".//td[@class='dddefault'][1]"])
def get_row_data(table):
   for row in table.find_elements_by_xpath(".//tr"):
        yield [td.text for td in row.find_elements_by_xpath(".//td[@class='dddefault'][text()]"])


for table in driver.find_elements_by_xpath("//table[@class='datadisplaytable']"):
    for data in get_row_data(table):
        # use the data
table = driver.find_element_by_xpath("//table[@class='datadisplaytable']")

for row in table.find_elements_by_xpath(".//tr"):
    print([td.text for td in row.find_elements_by_xpath(".//td[@class='dddefault']")])
#!/usr/bin/python

h  = """<table class="datadisplaytable">
<tr>
<td class="dddefault">16759</td>
<td class="dddefault">MATH</td>
<td class="dddefault">123</td>
<td class="dddefault">001</td>
<td class="dddefault">Calculus</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
<tr>
<td class="dddefault">16449</td>
<td class="dddefault">PHY</td>
<td class="dddefault">456</td>
<td class="dddefault">002</td>
<td class="dddefault">Physics</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
</table>"""

from lxml import html
xml = html.fromstring(h)
# gets the table
table =  xml.xpath("//table[@class='datadisplaytable']")[0]


# iterate over all the rows   
for row in table.xpath(".//tr"):
     # get the text from all the td's from each row
    print([td.text for td in row.xpath(".//td[@class='dddefault']")])

Selenium WebDriver - getthe first row of a table in C#

The below line gets the text of all visible rows. This works. IWebElement first_row = Driver.FindElement (By.CssSelector ("#headTable")); Another attempt to get the 1st row. This doesn't work. IWebElement first_row = Driver.FindElement (By.CssSelector ("#headTable")).FindElement (By.CssSelector ("tr:visible:first")); I've been looking at

IWebElement first_row = (RemoteWebElement)((IJavaScriptExecutor) Driver).ExecuteScript("return $('#headTable > tr:visible:first')");
IWebElement first_row = Driver.FindElement(By.CssSelector("#headTable"));
IWebElement first_row = Driver.FindElement(By.CssSelector("#headTable")).FindElement(By.CssSelector("tr:visible:first"));
WebElement table = Driver.FindElement(By.CssSelector("#headTable")); // you have the table
list<WebElement> allElements = table.findElements(By.CssSelector("tr[attribute name='visible']")); // you have all row that are visible
Table table = Table.Create(driver.FindElement(By.Id("headTable")));
TableRow row = table.FindRow(0);
string rowClasses = row.Element.GetAttribute("class");
Assert.IsTrue(rowClasses.Contains("expected-class"));

Previous PostNext Post

Related code examples