Một script để thu thập cơ sở dữ liệu của các diễn đàn. Python

pip install requests beautifulsoup4

import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import time

def is_external(url, base_url):
    return urlparse(url).netloc != urlparse(base_url).netloc

def get_links(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'}
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
        links = [a.get('href') for a in soup.find_all('a', href=True)]
        return links
    except requests.RequestException as e:
        print(f"Failed to fetch {url}: {e}")
        return []

def scrape_forums(starting_urls, max_depth=2):
    visited = set()
    external_links = set()
    
    def scrape(url, depth):
        if url in visited or depth > max_depth:
            return
        print(f"Scraping {url} at depth {depth}")
        visited.add(url)
        
        links = get_links(url)
        for link in links:
            full_url = urljoin(url, link)
            if is_external(full_url, url):
                external_links.add(full_url)
            else:
                scrape(full_url, depth + 1)
    
    for url in starting_urls:
        scrape(url, 1)
    
    return external_links

def save_links_to_file(links, filename):
    with open(filename, 'w') as f:
        for link in links:
            f.write(link + '\n')

if __name__ == '__main__':
    starting_urls = [
        # Add your starting forum URLs here
        'http://example-forum.com/topic1',
        'http://example-forum.com/topic2'
    ]
    filename = 'external_links.txt'
    
    external_links = scrape_forums(starting_urls)
    save_links_to_file(external_links, filename)
    
    print(f"Collected {len(external_links)} external links. Saved to {filename}.")

python script_name.py