How to scrape all stocks from screener website

Question

I want to extract all stocks names in CSV with python script, any one can provide me that?

Answer 1

import csv
import requests
from bs4 import BeautifulSoup

def scrape_page(url):
    # Send a GET request to the URL
    response = requests.get(url)

    # Check if the page exists
    if response.status_code != 200:
        return []

    # Parse the HTML content
    soup = BeautifulSoup(response.content, "html.parser")

    # Find the table containing the stock names
    table = soup.find("table", class_="data-table")

    if table:
        # Find all rows in the table
        rows = table.find_all("tr")

        # Extract stock names from each row
        stock_names = []
        for row in rows[1:]:  # Exclude header row
            cells = row.find_all("td")
            if cells:  # Check if cells exist in the row
                stock_name = cells[1].text.strip()  # Assuming the stock name is in the first column
                stock_names.append(stock_name)

        return stock_names
    else:
        print("Table not found on page:", url)
        return []

def scrape_all_pages(base_url):
    all_stock_names = []

    # Iterate over each page
    page_number = 1
    while True:
        page_url = base_url.format(page_number)
        print("Scraping page:", page_url)
        stock_names = scrape_page(page_url)
        if not stock_names:
            break
        all_stock_names.extend(stock_names)
        page_number += 1

    return all_stock_names

# URL of the webpage with pagination
base_url = "https://www.screener.in/screens/198573/all-stock-list-bse/?page={}"

# Scrape data from all pages
all_stock_names = scrape_all_pages(base_url)

# Write the stock names to a CSV file
csv_filename = "stock_names.csv"
with open(csv_filename, "w", newline="", encoding="utf-8") as csv_file:
    csv_writer = csv.writer(csv_file)
    csv_writer.writerow(["Stock Name"])  # Write header
    for name in all_stock_names:
        csv_writer.writerow([name])

print("All stock names have been scraped and saved in", csv_filename)