Scrapy के साथ शुरुआत

sudo apt install python3.11-venv
python3 -m venv venv
source venv/bin/activate
pip install scrapy

python3 -m venv venv
source venv/bin/activate
pip install scrapy

python -m venv venv
venv\Scripts\activate
pip install scrapy

scrapy startproject myproject
cd myproject
scrapy genspider sitename example.com

# पोलाइट क्रॉलिंग
CONCURRENT_REQUESTS_PER_DOMAIN = 5
DOWNLOAD_DELAY = 1
ROBOTSTXT_OBEY = True

# AutoThrottle - सर्वर रिस्पॉन्स के आधार पर स्पीड एडजस्ट करता है
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 5
AUTOTHROTTLE_MAX_DELAY = 60
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
AUTOTHROTTLE_DEBUG = True

# सेफ्टी लिमिट्स
CLOSESPIDER_PAGECOUNT = 10000

# आउटपुट
FEED_EXPORT_ENCODING = "utf-8"

handle_httpstatus_list = [200, 301, 302, 403, 404, 500, 502, 503]

import re
from urllib.parse import urlparse

class MySiteSpider(scrapy.Spider):
    name = "mysite"
    allowed_domains = ["example.com", "www.example.com"]
    start_urls = ["https://www.example.com/"]

    # /uk/, /fr/, /de/ जैसे इंटरनेशनल पाथ्स स्किप करें
    EXCLUDED_PATTERNS = re.compile(
        r"/(in|au|th|es|hk|sg|ph|my|ca|cn|uk|kr|id|fr|vn|de|jp|nl|it|tw)/"
    )

    def filter_links(self, links):
        filtered = []
        for link in links:
            hostname = urlparse(link.url).hostname or ""
            if hostname not in ("example.com", "www.example.com"):
                continue
            if self.EXCLUDED_PATTERNS.search(link.url):
                continue
            filtered.append(link)
        return filtered

def start_requests(self):
    # First, fetch robots.txt to find sitemaps
    yield Request(
        "https://www.example.com/robots.txt",
        callback=self.parse_robots,
        errback=self.handle_error,
        dont_filter=True,
    )
    # Also try common sitemap locations directly
    common_sitemaps = [
        "https://www.example.com/sitemap.xml",
        "https://www.example.com/sitemap_index.xml",
    ]
    for sitemap_url in common_sitemaps:
        yield Request(
            sitemap_url,
            callback=self.parse_sitemap,
            errback=self.handle_error,
            meta={"sitemap_url": sitemap_url},
        )
    # Also start normal crawl from homepage
    for url in self.start_urls:
        yield Request(url, callback=self.parse_start_url)

def parse_robots(self, response):
    """Parse robots.txt to find sitemap declarations"""
    if response.status != 200:
        return
    for line in response.text.splitlines():
        line = line.strip()
        if line.lower().startswith("sitemap:"):
            sitemap_url = line.split(":", 1)[1].strip()
            if self.is_valid_url(sitemap_url):
                self.logger.info(f"Found sitemap in robots.txt: {sitemap_url}")
                yield Request(
                    sitemap_url,
                    callback=self.parse_sitemap,
                    errback=self.handle_error,
                    meta={"sitemap_url": sitemap_url},
                )

def parse_sitemap(self, response):
    """Parse XML sitemap or sitemap index"""
    if response.status != 200:
        return
    content_type = response.headers.get("Content-Type", b"").decode("utf-8", errors="ignore")
    # Check if this is XML content
    if "xml" not in content_type and not response.text.strip().startswith("<?xml"):
        return
    # Check for sitemap index (contains other sitemaps)
    sitemap_locs = response.xpath("//sitemap/loc/text()").getall()
    if sitemap_locs:
        self.logger.info(f"Found sitemap index with {len(sitemap_locs)} sitemaps")
        for loc in sitemap_locs:
            if self.is_valid_url(loc):
                yield Request(
                    loc,
                    callback=self.parse_sitemap,
                    errback=self.handle_error,
                    meta={"sitemap_url": loc},
                )
    # Parse URL entries from sitemap
    url_locs = response.xpath("//url/loc/text()").getall()
    if url_locs:
        self.logger.info(f"Found {len(url_locs)} URLs in sitemap: {response.url}")
        for loc in url_locs:
            if self.is_valid_url(loc) and self.should_crawl_url(loc):
                yield Request(
                    loc,
                    callback=self.parse_page,
                    errback=self.handle_error,
                )

def parse_start_url(self, response):
    """Handle the start URL and trigger rules"""
    yield from self.parse_page(response)
    yield from self._requests_to_follow(response)

def is_valid_url(self, url):
    """Check if URL is valid and within allowed domains"""
    try:
        parsed = urlparse(url)
        hostname = parsed.hostname or ""
        return hostname in ("example.com", "www.example.com")
    except Exception:
        return False

def should_crawl_url(self, url):
    """Apply the same filtering as filter_links"""
    if self.EXCLUDED_PATTERNS.search(url):
        return False
    return True

def handle_error(self, failure):
    """Handle request errors gracefully"""
    self.logger.warning(f"Request failed: {failure.request.url}")

scrapy crawl myspider -o output.json -s JOBDIR=crawl_state

MEMUSAGE_LIMIT_MB = 1024
MEMUSAGE_WARNING_MB = 800
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'

def parse_page(self, response):
    yield {
        # बेसिक इन्फो
        "url": response.url,
        "status": response.status,
        "response_time": response.meta.get("download_latency"),

        # SEO फील्ड्स
        "title": response.css("title::text").get(),
        "meta_description": response.css('meta[name="description"]::attr(content)').get(),
        "h1": response.css("h1::text").get(),
        "h2_count": len(response.css("h2").getall()),

        # टेक्निकल
        "canonical": response.css('link[rel="canonical"]::attr(href)').get(),
        "robots": response.css('meta[name="robots"]::attr(content)').get(),
        "content_type": response.headers.get("Content-Type", b"").decode("utf-8"),
        "word_count": len(response.css("body *::text").getall()),

        # लिंक्स
        "internal_links": len([
            a for a in response.css("a::attr(href)").getall()
            if a.startswith("/") or "example.com" in a
        ]),
        "external_links": len([
            a for a in response.css("a::attr(href)").getall()
            if a.startswith("http") and "example.com" not in a
        ]),
    }

python3 -m venv venv
source venv/bin/activate  # Windows पर venv\Scripts\activate
pip install scrapy
scrapy startproject urlcrawler
cd urlcrawler
scrapy genspider mysite example.com
# पोलाइट क्रॉलिंग कॉन्फिग के साथ settings.py एडिट करें
# अपनी parse लॉजिक के साथ spiders/mysite.py एडिट करें
scrapy crawl mysite -o urls.jsonl -s JOBDIR=crawl_state

scrapy shell "https://example.com"

>>> response.css('title::text').get()
'Example Domain'
>>> response.xpath('//h1/text()').get()
'Example Domain'

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from urllib.parse import urlparse
import re


class SEOSpider(CrawlSpider):
    name = "seo_spider"
    allowed_domains = ["example.com", "www.example.com"]
    start_urls = ["https://www.example.com/"]

    # सभी स्टेटस कोड्स कैप्चर करें
    handle_httpstatus_list = [200, 301, 302, 403, 404, 500, 502, 503]

    # इंटरनेशनल पाथ्स एक्सक्लूड करें
    EXCLUDED_PATTERNS = re.compile(
        r"/(in|au|th|es|hk|sg|ph|my|ca|cn|uk|kr|id|fr|vn|de|jp|nl|it|tw)/"
    )

    rules = (
        Rule(
            LinkExtractor(
                allow_domains=["example.com", "www.example.com"],
                deny=[r"\?", r"#"],  # क्वेरी स्ट्रिंग्स और फ्रैगमेंट्स स्किप करें
            ),
            callback="parse_page",
            follow=True,
        ),
    )

    def parse_page(self, response):
        # एक्सक्लूडेड पाथ्स स्किप करें
        if self.EXCLUDED_PATTERNS.search(response.url):
            return

        yield {
            # बेसिक इन्फो
            "url": response.url,
            "status": response.status,
            "response_time": response.meta.get("download_latency"),

            # SEO फील्ड्स
            "title": response.css("title::text").get(),
            "meta_description": response.css(
                'meta[name="description"]::attr(content)'
            ).get(),
            "h1": response.css("h1::text").get(),
            "h2_count": len(response.css("h2").getall()),

            # टेक्निकल
            "canonical": response.css('link[rel="canonical"]::attr(href)').get(),
            "robots": response.css('meta[name="robots"]::attr(content)').get(),
            "content_type": response.headers.get("Content-Type", b"").decode("utf-8"),
            "word_count": len(response.css("body *::text").getall()),

            # लिंक्स
            "internal_links": len([
                a for a in response.css("a::attr(href)").getall()
                if a.startswith("/") or "example.com" in a
            ]),
            "external_links": len([
                a for a in response.css("a::attr(href)").getall()
                if a.startswith("http") and "example.com" not in a
            ]),
        }

scrapy crawl seo_spider -o results.jsonl -s JOBDIR=crawl_state