We provide programming data of 20 most popular languages, hope to help you!
But the result is missing the first row (2021-11) and the last column (净投放(亿)). What have I done wrong? from selenium import webdriver driver = Selenium loop, table missing first row and last column. Ask Question Asked 7 months ago. Have updated the code. Scroll down to footer and then try to extract the details – pmadhu. Nov
from selenium import webdriver
driver = webdriver.Firefox(executable_path = '/usr/local/bin/geckodriver')
driver.get('http://www.chinamoney.com.cn/chinese/hb/')
rws = driver.find_elements_by_xpath("//table/tbody/tr")
r = len(rws)
cols = driver.find_elements_by_xpath("//thead/tr/td")
c = len(cols)
element = []
row = []
for i in range(1,r):
for j in range(1,c):
d=driver.find_element_by_xpath("//tr["+str(i)+"]/td["+str(j)+"]").text
row.append(d)
element.append(row)
driver.close()
element
driver.get("http://www.chinamoney.com.cn/chinese/hb/")
# Scroll Down to footer so that the all the data in the table is accessible to selenium
driver.execute_script("arguments[0].scrollIntoView(true);",driver.find_element_by_id("main-page-foot"))
# Collect the table rows using - xpath "//table//tr", in the variable rows
rows = driver.find_elements_by_xpath("//table//tr") # rows is a list of webelements - rows of the table(tr tags of the table).
element = []
# Iterate over the list
for row in rows:
# Get the data inside a row. Columns is also a list of webelements - "td" tags of that particular "tr"
columns = row.find_elements_by_xpath(".//td") # Put a dot in the xpath to get an element from an element.
list_row = [] # Temporary list to store values of td tags
for col in columns:
list_row.append(col.text) # Append all the td tags values
element.append(list_row) # Append the list of td tag values
print(element)
[['日期', '投放量(亿)', '回笼量(亿)', '净投放(亿)'], ['2021-11', '2200', '12200', '-10000'], ['2021-10', '13900', '12300', '1600'], ['2021-09', '11800', '5900', '5900'], ['2021-08', '4200', '2600', '1600'], ['2021-07', '2600', '3200', '-600'], ['2021-06', '3100', '2100', '1000'], ['2021-05', '1900', '2000', '-100'], ['2021-04', '2200', '2100', '100'], ['2021-03', '2400', '2500', '-100'], ['2021-02', '8300', '11440', '-3140'], ['2021-01', '10740', '12450', '-1710'], ['2020-12', '9150', '8900', '250'], ['2020-11', '17550', '17200', '350'], ['', '', '', ''], ['', '', '', '']]
from selenium import webdriver
driver = webdriver.Firefox(executable_path = '/usr/local/bin/geckodriver')
driver.get('http://www.chinamoney.com.cn/chinese/hb/')
time.sleep(5)
rws = driver.find_elements_by_xpath("//table/tbody/tr")
r = len(rws)
cols = driver.find_elements_by_xpath("//thead/tr/td")
c = len(cols)
element = []
row = []
for i in range(r):
for j in range(c):
d=driver.find_element_by_xpath("//tr["+str(i)+"]/td["+str(j)+"]").text
row.append(d)
element.append(row)
driver.close()
element
And I managed to scrape it but it resulted in only the first row of the table instead of 20 rows. I saw previous similar questions that were answered and I have tried the solutions given but my selenium was unable to locate the element when I use .// for my xpath.
for bod in driver.find_elements_by_xpath("//*[@id='proxylisttable']/tbody"):
col = bod.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr")
for c in col:
ip = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[1]')
port = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[2]')
code = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[3]')
country = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[4][@class = "hm"]')
anonymity = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[5]')
google = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[6][@class = "hm"]')
col = bod.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr")
col = bod.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr[insert count here]")
table = driver.find_element_by_xpath("//*[@id='proxylisttable']/tbody")
rows = table.find_elements_by_xpath("//*[@id='proxylisttable']/tbody/tr")
for i in range (1, len(rows)+1):
row = table.find_element_by_xpath("//*[@id='proxylisttable']/tbody/tr[" +str(i) +']')
for c in row:
ip = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[1]')
port = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[2]')
code = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[3]')
country = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[4][@class = "hm"]')
anonymity = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[5]')
google = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr/td[6][@class = "hm"]')
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver=webdriver.Chrome("path of the chrome driver")
driver.get('https://free-proxy-list.net/')
rows= WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//table[@id='proxylisttable']/tbody//tr")))
for row in rows:
ip=row.find_element_by_xpath('./td[1]').text
port=row.find_element_by_xpath('./td[2]').text
code=row.find_element_by_xpath('./td[3]').text
country=row.find_element_by_xpath('./td[4]').get_attribute('textContent')
Anonymity=row.find_element_by_xpath('./td[5]').text
google=row.find_element_by_xpath('./td[6]').get_attribute('textContent')
https=row.find_element_by_xpath('./td[7]').text
lastchecked=row.find_element_by_xpath('./td[8]').get_attribute('textContent')
print("IP :{}, Port:{}, code:{}, country:{}, Anonymity:{}, google:{}, https:{}, last_checked:{}".format(ip,port,code,country,Anonymity,google,https,lastchecked))
IP :185.132.133.173, Port:8080, code:NL, country:Netherlands, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :181.112.225.78, Port:58948, code:EC, country:Ecuador, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :134.249.149.219, Port:35795, code:UA, country:Ukraine, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :195.20.30.54, Port:55182, code:UA, country:Ukraine, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :14.102.69.170, Port:53347, code:IN, country:India, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :182.53.193.108, Port:54543, code:TH, country:Thailand, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :159.224.221.175, Port:58299, code:UA, country:Ukraine, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :36.89.188.123, Port:49725, code:ID, country:Indonesia, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :103.231.163.58, Port:43620, code:BD, country:Bangladesh, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :114.130.92.14, Port:49167, code:BD, country:Bangladesh, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :177.54.200.10, Port:49501, code:BR, country:Brazil, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :77.38.21.239, Port:8080, code:SI, country:Slovenia, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :78.137.89.161, Port:8080, code:YE, country:Yemen, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :103.216.147.49, Port:8080, code:IN, country:India, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :195.250.188.210, Port:8080, code:EE, country:Estonia, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :5.196.255.171, Port:3128, code:FR, country:France, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :109.234.112.250, Port:46675, code:GE, country:Georgia, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :186.225.48.178, Port:8080, code:BR, country:Brazil, Anonymity:transparent, google:no, https:no, last_checked:1 minute ago
IP :101.255.64.142, Port:35401, code:ID, country:Indonesia, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
IP :160.119.129.42, Port:57557, code:GN, country:Guinea, Anonymity:elite proxy, google:no, https:yes, last_checked:1 minute ago
ip = c.find_element_by_xpath('//*[@id="proxylisttable"]/tbody/tr[index]/td[1]')
The problem is that the webtable is dynamic so I don't know the row number.I have tried below code but the row number is static there whic Stack Overflow. About; In first Line we get the Last Row count from the table. On second line we get the Last row Element from the table. Selenium cannot handle large table row count. 0.
List<WebElement> rows=driver.findElements(By.xpath("//table[1]/tbody/tr[5]"));
WebElement lastRow = driver.findElement(By.xpath("(//table[1]/tbody/tr)[last()]"));
(locator)[position() = last()]
public class LastRowFetching {
public static void main(String[] args) throws ParseException {
WebDriver driver;
System.setProperty("webdriver.chrome.driver","G://chromedriver.exe");
driver= new ChromeDriver();
driver.get("http://money.rediff.com/gainers/bsc/dailygroupa?");
//No.of rows
List rows = driver.findElements(By.xpath(".//*[@id='leftcontainer']/table/tbody/tr/td[1]"));
System.out.println("No of rows are : " + rows.size());
int lastRow = rows.size();
WebElement lastRowFetch =driver.findElement(By.xpath(".//*[@id='leftcontainer']/table/tbody/tr/td["+lastRowcount+"])"));
driver.close();
}
}
raise exception_class(message, screen, stacktrace)
StaleElementReferenceException: Message: Element is no longer valid
col_name = row.find_elements(By.TAG_NAME, "td")[1] # This is the Name column
def get_variables_col_values(self):
try:
table_id = self.driver.find_element(By.ID, 'data_configuration_variables_ct_fields_body1')
#time.sleep(10)
rows = table_id.find_elements(By.TAG_NAME, "tr")
print "Rows length"
print len(rows)
for row in rows:
# Get the columns
print "cols length"
print len(row.find_elements(By.TAG_NAME, "td"))
col_name = row.find_elements(By.TAG_NAME, "td")[1] # This is the Name column print "col_name.text = "
print col_name.text
except NoSuchElementException, e:
return False
WebDriverWait(self.driver, 10).until(lambda d: d.execute_script('return document.readyState') == 'complete')
Rows length
16
cols length
6
col_name.text =
Name
cols length
6
col_name.text =
Address
cols length
6
col_name.text =
DOB
...
from selenium.webdriver.support import expected_conditions as EC
def get_variables_col_values(self):
try:
WebDriverWait(self.driver, 10).until(EC.presence_of_all_elements_located((By.TAG_NAME,'td')))
table_id = self.driver.find_element(By.ID, 'data_configuration_variables_ct_fields_body1')
#time.sleep(10)
rows = table_id.find_elements(By.TAG_NAME, "tr")
print "Rows length"
print len(rows)
for row in rows:
# Get the columns
print "cols length"
print len(row.find_elements(By.TAG_NAME, "td"))
col_name = row.find_elements(By.TAG_NAME, "td")[1] # This is the Name column print "col_name.text = "
print col_name.text
except NoSuchElementException, e:
return False
Iterating Through Table Rows in Selenium (Python) Ask Question Asked 6 years, My function always seems to go with the first table with that name, when I'm looking for the next one (which is the one below it). there was one missing round bracket at the end; also removed the [1] index, to match the first XML example.
<table class="datadisplaytable">
<tbody>
<tr>
<td class="dddefault">16759</td>
<td class="dddefault">MATH</td>
<td class="dddefault">123</td>
<td class="dddefault">001</td>
<td class="dddefault">Calculus</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
<tr>
<td class="dddefault">16449</td>
<td class="dddefault">PHY</td>
<td class="dddefault">456</td>
<td class="dddefault">002</td>
<td class="dddefault">Physics</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
</tbody>
</table>
def check_grades(self):
table = []
for i in self.driver.find_element_by_class_name("dddefault"):
table.append(i)
print(table)
mytable = find_element_by_css_selector('table.datadisplaytable')
for row in mytable.find_elements_by_css_selector('tr'):
for cell in row.find_elements_by_tag_name('td'):
print(cell.text)
h = """<table class="datadisplaytable">
<tr>
<td class="dddefault">16759</td>
<td class="dddefault">MATH</td>
<td class="dddefault">123</td>
<td class="dddefault">001</td>
<td class="dddefault">Calculus</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
<tr>
<td class="dddefault">16449</td>
<td class="dddefault">PHY</td>
<td class="dddefault">456</td>
<td class="dddefault">002</td>
<td class="dddefault">Physics</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
</table>"""
from lxml import html
xml = html.fromstring(h)
# gets the table
table = xml.xpath("//table[@class='datadisplaytable']")[0]
# iterate over all the rows
for row in table.xpath(".//tr"):
# get the text from all the td's from each row
print([td.text for td in row.xpath(".//td[@class='dddefault'][text()])
['16759', 'MATH', '123', '001', 'Calculus']
['16449', 'PHY', '456', '002', 'Physics']
table = driver.find_element_by_xpath("//table[@class='datadisplaytable']")
for row in table.find_elements_by_xpath(".//tr"):
print([td.text for td in row.find_elements_by_xpath(".//td[@class='dddefault'][1]"])
def get_row_data(table):
for row in table.find_elements_by_xpath(".//tr"):
yield [td.text for td in row.find_elements_by_xpath(".//td[@class='dddefault'][text()]"])
for table in driver.find_elements_by_xpath("//table[@class='datadisplaytable']"):
for data in get_row_data(table):
# use the data
table = driver.find_element_by_xpath("//table[@class='datadisplaytable']")
for row in table.find_elements_by_xpath(".//tr"):
print([td.text for td in row.find_elements_by_xpath(".//td[@class='dddefault']")])
#!/usr/bin/python
h = """<table class="datadisplaytable">
<tr>
<td class="dddefault">16759</td>
<td class="dddefault">MATH</td>
<td class="dddefault">123</td>
<td class="dddefault">001</td>
<td class="dddefault">Calculus</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
<tr>
<td class="dddefault">16449</td>
<td class="dddefault">PHY</td>
<td class="dddefault">456</td>
<td class="dddefault">002</td>
<td class="dddefault">Physics</td>
<td class="dddefault"></td>
<td class="dddead"></td>
<td class="dddead"></td>
</tr>
</table>"""
from lxml import html
xml = html.fromstring(h)
# gets the table
table = xml.xpath("//table[@class='datadisplaytable']")[0]
# iterate over all the rows
for row in table.xpath(".//tr"):
# get the text from all the td's from each row
print([td.text for td in row.xpath(".//td[@class='dddefault']")])
The below line gets the text of all visible rows. This works. IWebElement first_row = Driver.FindElement (By.CssSelector ("#headTable")); Another attempt to get the 1st row. This doesn't work. IWebElement first_row = Driver.FindElement (By.CssSelector ("#headTable")).FindElement (By.CssSelector ("tr:visible:first")); I've been looking at
IWebElement first_row = (RemoteWebElement)((IJavaScriptExecutor) Driver).ExecuteScript("return $('#headTable > tr:visible:first')");
IWebElement first_row = Driver.FindElement(By.CssSelector("#headTable"));
IWebElement first_row = Driver.FindElement(By.CssSelector("#headTable")).FindElement(By.CssSelector("tr:visible:first"));
WebElement table = Driver.FindElement(By.CssSelector("#headTable")); // you have the table
list<WebElement> allElements = table.findElements(By.CssSelector("tr[attribute name='visible']")); // you have all row that are visible
Table table = Table.Create(driver.FindElement(By.Id("headTable")));
TableRow row = table.FindRow(0);
string rowClasses = row.Element.GetAttribute("class");
Assert.IsTrue(rowClasses.Contains("expected-class"));