Een script voor het verzamelen van een database met forums. Python

pip install requests beautifulsoup4

import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import time

def is_external(url, base_url):
    return urlparse(url).netloc != urlparse(base_url).netloc

def get_links(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'}
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'html.parser')
        links = [a.get('href') for a in soup.find_all('a', href=True)]
        return links
    except requests.RequestException as e:
        print(f"Failed to fetch {url}: {e}")
        return []

def scrape_forums(starting_urls, max_depth=2):
    visited = set()
    external_links = set()
    
    def scrape(url, depth):
        if url in visited or depth > max_depth:
            return
        print(f"Scraping {url} at depth {depth}")
        visited.add(url)
        
        links = get_links(url)
        for link in links:
            full_url = urljoin(url, link)
            if is_external(full_url, url):
                external_links.add(full_url)
            else:
                scrape(full_url, depth + 1)
    
    for url in starting_urls:
        scrape(url, 1)
    
    return external_links

def save_links_to_file(links, filename):
    with open(filename, 'w') as f:
        for link in links:
            f.write(link + '\n')

if __name__ == '__main__':
    starting_urls = [
        # Add your starting forum URLs here
        'http://example-forum.com/topic1',
        'http://example-forum.com/topic2'
    ]
    filename = 'external_links.txt'
    
    external_links = scrape_forums(starting_urls)
    save_links_to_file(external_links, filename)
    
    print(f"Collected {len(external_links)} external links. Saved to {filename}.")

python script_name.py