Початок роботи зі Scrapy

sudo apt install python3.11-venv
python3 -m venv venv
source venv/bin/activate
pip install scrapy

python3 -m venv venv
source venv/bin/activate
pip install scrapy

python -m venv venv
venv\Scripts\activate
pip install scrapy

scrapy startproject myproject
cd myproject
scrapy genspider sitename example.com

# Ввічливий краулінг
CONCURRENT_REQUESTS_PER_DOMAIN = 5
DOWNLOAD_DELAY = 1
ROBOTSTXT_OBEY = True

# AutoThrottle - регулює швидкість на основі відповіді сервера
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 5
AUTOTHROTTLE_MAX_DELAY = 60
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
AUTOTHROTTLE_DEBUG = True

# Ліміти безпеки
CLOSESPIDER_PAGECOUNT = 10000

# Вивід
FEED_EXPORT_ENCODING = "utf-8"

handle_httpstatus_list = [200, 301, 302, 403, 404, 500, 502, 503]

import re
from urllib.parse import urlparse

class MySiteSpider(scrapy.Spider):
    name = "mysite"
    allowed_domains = ["example.com", "www.example.com"]
    start_urls = ["https://www.example.com/"]

    # Пропускаємо міжнародні шляхи типу /uk/, /fr/, /de/
    EXCLUDED_PATTERNS = re.compile(
        r"/(in|au|th|es|hk|sg|ph|my|ca|cn|uk|kr|id|fr|vn|de|jp|nl|it|tw)/"
    )

    def filter_links(self, links):
        filtered = []
        for link in links:
            hostname = urlparse(link.url).hostname or ""
            if hostname not in ("example.com", "www.example.com"):
                continue
            if self.EXCLUDED_PATTERNS.search(link.url):
                continue
            filtered.append(link)
        return filtered

def start_requests(self):
    # First, fetch robots.txt to find sitemaps
    yield Request(
        "https://www.example.com/robots.txt",
        callback=self.parse_robots,
        errback=self.handle_error,
        dont_filter=True,
    )
    # Also try common sitemap locations directly
    common_sitemaps = [
        "https://www.example.com/sitemap.xml",
        "https://www.example.com/sitemap_index.xml",
    ]
    for sitemap_url in common_sitemaps:
        yield Request(
            sitemap_url,
            callback=self.parse_sitemap,
            errback=self.handle_error,
            meta={"sitemap_url": sitemap_url},
        )
    # Also start normal crawl from homepage
    for url in self.start_urls:
        yield Request(url, callback=self.parse_start_url)

def parse_robots(self, response):
    """Parse robots.txt to find sitemap declarations"""
    if response.status != 200:
        return
    for line in response.text.splitlines():
        line = line.strip()
        if line.lower().startswith("sitemap:"):
            sitemap_url = line.split(":", 1)[1].strip()
            if self.is_valid_url(sitemap_url):
                self.logger.info(f"Found sitemap in robots.txt: {sitemap_url}")
                yield Request(
                    sitemap_url,
                    callback=self.parse_sitemap,
                    errback=self.handle_error,
                    meta={"sitemap_url": sitemap_url},
                )

def parse_sitemap(self, response):
    """Parse XML sitemap or sitemap index"""
    if response.status != 200:
        return
    content_type = response.headers.get("Content-Type", b"").decode("utf-8", errors="ignore")
    # Check if this is XML content
    if "xml" not in content_type and not response.text.strip().startswith("<?xml"):
        return
    # Check for sitemap index (contains other sitemaps)
    sitemap_locs = response.xpath("//sitemap/loc/text()").getall()
    if sitemap_locs:
        self.logger.info(f"Found sitemap index with {len(sitemap_locs)} sitemaps")
        for loc in sitemap_locs:
            if self.is_valid_url(loc):
                yield Request(
                    loc,
                    callback=self.parse_sitemap,
                    errback=self.handle_error,
                    meta={"sitemap_url": loc},
                )
    # Parse URL entries from sitemap
    url_locs = response.xpath("//url/loc/text()").getall()
    if url_locs:
        self.logger.info(f"Found {len(url_locs)} URLs in sitemap: {response.url}")
        for loc in url_locs:
            if self.is_valid_url(loc) and self.should_crawl_url(loc):
                yield Request(
                    loc,
                    callback=self.parse_page,
                    errback=self.handle_error,
                )

def parse_start_url(self, response):
    """Handle the start URL and trigger rules"""
    yield from self.parse_page(response)
    yield from self._requests_to_follow(response)

def is_valid_url(self, url):
    """Check if URL is valid and within allowed domains"""
    try:
        parsed = urlparse(url)
        hostname = parsed.hostname or ""
        return hostname in ("example.com", "www.example.com")
    except Exception:
        return False

def should_crawl_url(self, url):
    """Apply the same filtering as filter_links"""
    if self.EXCLUDED_PATTERNS.search(url):
        return False
    return True

def handle_error(self, failure):
    """Handle request errors gracefully"""
    self.logger.warning(f"Request failed: {failure.request.url}")

scrapy crawl myspider -o output.json -s JOBDIR=crawl_state

MEMUSAGE_LIMIT_MB = 1024
MEMUSAGE_WARNING_MB = 800
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'

{
    "url": "https://www.example.com/page/",
    "title": "Заголовок сторінки тут",
    "status": 200
}

def parse_page(self, response):
    yield {
        "url": response.url,
        "status": response.status,
        "title": response.css("title::text").get(),
        "meta_description": response.css("meta[name='description']::attr(content)").get(),
        "meta_robots": response.css("meta[name='robots']::attr(content)").get(),
        "h1": response.css("h1::text").get(),
        "canonical": response.css("link[rel='canonical']::attr(href)").get(),
        "og_title": response.css("meta[property='og:title']::attr(content)").get(),
        "og_description": response.css("meta[property='og:description']::attr(content)").get(),
        "word_count": len(response.text.split()) if response.status == 200 else None,
        "content_type": response.headers.get("Content-Type", b"").decode("utf-8", errors="ignore"),
    }

python3 -m venv venv
source venv/bin/activate  # venv\Scripts\activate на Windows
pip install scrapy
scrapy startproject urlcrawler
cd urlcrawler
scrapy genspider mysite example.com
# Відредагуйте settings.py з конфігурацією ввічливого краулу
# Відредагуйте spiders/mysite.py з вашою логікою parse
scrapy crawl mysite -o urls.jsonl -s JOBDIR=crawl_state

scrapy shell "https://example.com"

>>> response.css('title::text').get()
'Example Domain'
>>> response.xpath('//h1/text()').get()
'Example Domain'

import re
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from urllib.parse import urlparse


class SEOSpider(CrawlSpider):
    name = "seospider"
    allowed_domains = ["example.com"]
    start_urls = ["https://www.example.com"]

    # Capture all HTTP status codes, not just 2xx
    handle_httpstatus_list = [200, 301, 302, 403, 404, 500, 502, 503]

    # URL patterns to exclude
    EXCLUDED_PATTERNS = re.compile(
        r"/(in|au|th|es|hk|sg|ph|my|ca|cn|uk|kr|id|fr|vn|de|jp|nl|it|tw)/"
    )

    rules = (
        Rule(
            LinkExtractor(allow=()),
            callback="parse_page",
            follow=True,
            process_links="filter_links",
        ),
    )

    def filter_links(self, links):
        filtered = []
        for link in links:
            parsed = urlparse(link.url)
            hostname = parsed.hostname or ""

            if hostname not in ("example.com", "www.example.com"):
                continue

            if self.EXCLUDED_PATTERNS.search(link.url):
                continue

            filtered.append(link)
        return filtered

    def parse_page(self, response):
        yield {
            "url": response.url,
            "status": response.status,
            "title": response.css("title::text").get(),
            "meta_description": response.css("meta[name='description']::attr(content)").get(),
            "meta_robots": response.css("meta[name='robots']::attr(content)").get(),
            "h1": response.css("h1::text").get(),
            "canonical": response.css("link[rel='canonical']::attr(href)").get(),
            "og_title": response.css("meta[property='og:title']::attr(content)").get(),
            "og_description": response.css("meta[property='og:description']::attr(content)").get(),
            "word_count": len(response.text.split()) if response.status == 200 else None,
            "content_type": response.headers.get("Content-Type", b"").decode("utf-8", errors="ignore"),
        }