# WebDirectory.it - Enhanced Robots.txt
# Updated for current website structure and SEO optimization

User-agent: *
Allow: /
Allow: /category/
Allow: /assets/css/style.css
Allow: /assets/js/main.js

# Block admin and sensitive files
Disallow: /cache_backups/
Disallow: /xynztya.php
Disallow: /add_domain.php
Disallow: /*admin*
Disallow: /*cache*
Disallow: /*.json$
Disallow: /*.log$
Disallow: /error.log

# Block unnecessary asset directories from indexing
Disallow: /assets/

# Allow important static files
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /sitemap.php
Allow: /404.html
Allow: /500.html

# Crawl delay for better server performance
Crawl-delay: 2

# Sitemap locations
Sitemap: https://webdirectory.it.com/sitemap.xml
Sitemap: https://webdirectory.it.com/sitemap.php

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Allow: /category/
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Allow: /category/
Crawl-delay: 2

User-agent: Slurp
Allow: /
Allow: /category/
Crawl-delay: 3

# Block aggressive crawlers and scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MajesticSEO
Disallow: /

User-agent: BLEXBot
Disallow: /

# Allow social media crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /