{ "version": "6.64", "updated": "2026-10-14T06:03:29+03:00", "site": "https://mail.solidmedya.com", "host": "mail.solidmedya.com", "policy": "allow-all-public-crawlers", "blocking": false, "managed_robots_txt": "OFF — Cloudflare «Managed robots.txt» kapalı olmalı (Content-Signal GSC uyarısı verir)", "gsc_robots_warnings": [ "Content-Signal robots.txt içinde standart değildir; Google Search Console yok sayar.", "Cloudflare panel: Scrape Shield / Content Signals / Managed robots.txt özelliğini kapatın veya site kökenini hariç tutun.", "AI tercihleri bu dosyada User-agent: GPTBot, Google-Extended vb. bloklarla yönetilir." ], "notes": [ "Bot Fight Mode (ücretsiz plan) özel kurallarla atlanamaz; doğrulanmış arama ve AI botlarına zaten dokunmaz.", "Super Bot Fight Mode: \"Verified bots\" = Allow; aşağıdaki atlama kuralları yalnız SBFM içindir.", "AI Crawl Control: Search ve Agent botları izinli kalsın; Training engeli Googlebot/Bingbot gibi çok amaçlı botları da etkileyebilir." ], "rules": [ { "name": "Skip Super Bot Fight — keşif yolları", "action": "skip", "phases": [ "http_request_sbfm" ], "expression": "(http.host eq \"mail.solidmedya.com\" and (http.request.uri.path starts_with \"/.well-known/\" or http.request.uri.path starts_with \"/content/\" or http.request.uri.path in {\"/llms.txt\" \"/llms\" \"/ai.txt\" \"/robots.txt\" \"/sitemap.xml\" \"/feed.xml\" \"/rss.xml\"}))" }, { "name": "Skip Super Bot Fight — doğrulanmış arama ve AI botları", "action": "skip", "phases": [ "http_request_sbfm" ], "expression": "(http.host eq \"mail.solidmedya.com\" and (cf.client.bot or cf.verified_bot_category in {\"Search Engine Crawler\" \"AI Search\" \"AI Assistant\" \"AI Crawler\"}))" } ], "discovery_paths": [ "/.well-known/*", "/content/*", "/llms.txt", "/llms", "/ai.txt", "/robots.txt", "/sitemap.xml", "/feed.xml", "/rss.xml" ], "allowed_agents": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "Google-Extended", "Googlebot", "Googlebot-Image", "Googlebot-Video", "GoogleOther", "ClaudeBot", "Claude-SearchBot", "Claude-User", "anthropic-ai", "Claude-Web", "PerplexityBot", "Perplexity-User", "Applebot", "Applebot-Extended", "cohere-ai", "FacebookBot", "Meta-ExternalAgent", "Meta-ExternalFetcher", "Bytespider", "YouBot", "Amazonbot", "CCBot", "Diffbot", "ImagesiftBot", "Omgilibot", "Omgili", "PetalBot", "Timpibot", "Webzio-Extended", "MistralAI-User", "AI2Bot", "Google-CloudVertexBot", "GrokBot", "xAI-Grok", "DeepSeekBot", "DuckAssistBot", "LinkedInBot", "Twitterbot", "facebookexternalhit", "ia_archiver", "Bingbot", "msnbot", "YandexBot", "Yandex", "DuckDuckBot", "Baiduspider", "Sogou", "SeznamBot", "NaverBot", "Yahoo! Slurp", "Qwantify", "Ecosia", "Facebot", "WhatsApp", "TelegramBot", "Slackbot", "Discordbot", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot", "Screaming Frog SEO Spider" ], "robots": { "$schema": "https://mail.solidmedya.com/.well-known/robots.schema.json", "spec": "RFC 9309", "version": "2.2", "updated": "2026-10-14T06:03:29+03:00", "generator": "Solid Medya Robots v6.64", "summary": "Tam arama + AI atıf erişimi; eğitim optional; 66 izinli UA, 14 engelli scraper.", "url": "https://mail.solidmedya.com/robots.txt", "policy": { "search": "allow", "ai_citation": "allow", "ai_training": "optional", "blocking": false, "citation": "Solid Medya (solidmedya.com) — [Başlık]" }, "content_signal": { "search": "yes", "ai_input": "yes", "ai_train": "optional" }, "sitemaps": [ "https://mail.solidmedya.com/sitemap.xml", "https://mail.solidmedya.com/sitemap-images.xml", "https://mail.solidmedya.com/sitemap-videos.xml" ], "bot_categories": { "search_engines": [ "Bingbot", "msnbot", "YandexBot", "Yandex", "DuckDuckBot", "Baiduspider", "Sogou", "SeznamBot", "NaverBot", "Yahoo! Slurp", "Qwantify", "Ecosia" ], "ai_citation": [ "OAI-SearchBot", "ChatGPT-User", "PerplexityBot", "Perplexity-User", "Claude-SearchBot", "Claude-User", "DuckAssistBot", "MistralAI-User" ], "ai_training": [ "GPTBot", "Google-Extended", "ClaudeBot", "anthropic-ai", "Claude-Web", "Applebot", "Applebot-Extended", "cohere-ai", "FacebookBot", "Meta-ExternalAgent", "Meta-ExternalFetcher", "Bytespider", "YouBot", "Amazonbot", "CCBot", "Diffbot", "ImagesiftBot", "Omgilibot", "Omgili", "PetalBot", "Timpibot", "Webzio-Extended", "AI2Bot", "Google-CloudVertexBot", "GrokBot", "xAI-Grok", "DeepSeekBot", "LinkedInBot", "Twitterbot", "facebookexternalhit", "ia_archiver" ], "social_tools": [ "facebookexternalhit", "Facebot", "Twitterbot", "LinkedInBot", "WhatsApp", "TelegramBot", "Slackbot", "Discordbot", "AhrefsBot", "SemrushBot", "MJ12bot", "DotBot", "Screaming Frog SEO Spider" ], "blocked_scrapers": [ "HTTrack", "WebCopier", "WebZIP", "Teleport", "EmailCollector", "EmailSiphon", "WebBandit", "Exabot", "ExtractorPro", "CherryPicker", "Nutch", "TurnitinBot", "SiteSnagger", "ProWebWalker" ] }, "disallow_paths": [ "/admin/", "/administrator/", "/private/", "/panel/", "/login/", "/giris", "/giris/", "/session/", "/app/", "/api/", "/config/", "/storage/", "/storage/logs/", "/cache/", "/temp/", "/tmp/", "/backup/", "/vendor/", "/node_modules/", "/.git/", "/eposta/", "/kart/", "/public/", "/system/", "/test/", "/dev/", "/beta/", "/staging/", "/sitemaps/", "/api/sitemaps/", "/digeryazilimlar/", "/yasinakbiyik/", "/cron/" ], "noindex_paths": [ "/arama", "/arama/" ], "parameter_disallow": [ "/*?print=", "/*&print=", "/*?preview=", "/*&preview=", "/*?debug=", "/*&debug=", "/*?test=", "/*&test=", "/*?pwa=true", "/*&pwa=true", "/*ref=affiliate", "/*ref=partner" ], "statistics": { "bytes": 15625, "user_agent_blocks": 80, "sitemap_count": 3, "disallow_rules": 433, "allow_rules": 65 }, "validation": { "status": "gaps", "total_issues": 2 }, "discovery": { "llms_txt": "https://mail.solidmedya.com/llms.txt", "ai_txt": "https://mail.solidmedya.com/ai.txt", "indexing_json": "https://mail.solidmedya.com/.well-known/indexing.json", "entity_graph": "https://mail.solidmedya.com/.well-known/entity-graph.json", "indexnow": "https://mail.solidmedya.com/.well-known/indexnow.json", "bot_access": "https://mail.solidmedya.com/.well-known/bot-access.json" }, "usage_guide": [ "1. Birincil dosya: GET https://mail.solidmedya.com/robots.txt", "2. AI atıf botları (OAI-SearchBot, PerplexityBot) açık kalmalı", "3. GPTBot=eğitim, OAI-SearchBot=ChatGPT arama — ayrı kurallar", "4. noindex: /arama — site içi arama" ], "docs": { "rfc9309": "https://www.rfc-editor.org/rfc/rfc9309.html", "google": "https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt", "ai_crawl": "https://developers.openai.com/api/docs/bots" } }, "live_probe": null, "endpoints": { "well_known_index": "https://mail.solidmedya.com/.well-known/index.json", "discovery": "https://mail.solidmedya.com/.well-known/discovery.json", "indexing": "https://mail.solidmedya.com/.well-known/indexing.json", "entity_graph": "https://mail.solidmedya.com/.well-known/entity-graph.json", "indexnow": "https://mail.solidmedya.com/.well-known/indexnow.json", "llms": "https://mail.solidmedya.com/.well-known/llms.json", "ai_txt": "https://mail.solidmedya.com/ai.txt", "robots": "https://mail.solidmedya.com/robots.txt", "robots_json": "https://mail.solidmedya.com/.well-known/robots.json", "health": "https://mail.solidmedya.com/.well-known/health.json" } }