User-agent: *


# Directories
Disallow: /404/
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pub/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# Paths (Clean URLs)
Disallow: /index.php/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /control/
Disallow: /contacts/
Disallow: /customer/
Disallow: /customize/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /review/
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/

# Files
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /CONTRIBUTING.md
Disallow: /CONTRIBUTORS.md
Disallow: /Gruntfile.js
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /nginx.conf.sample
Disallow: /package.json
Disallow: /php.ini.sample
Disallow: /RELEASE_NOTES.txt

# Parameters
Disallow: /*?dir*
Disallow: /*?dir=asc
Disallow: /*?dir=desc
Disallow: /*?mode*
Disallow: /*?limit*
Disallow: /*?order*
Disallow: /*?price*
Disallow: /*?cat=
Disallow: /*&cat=
Disallow: /*product_type_1
Disallow: /*?brand=
Disallow: /*&brand=

# Sitemaps
Sitemap: https://servair.com/sitemap.xml
Sitemap: https://servair.com/blog-sitemap.xml

# --- Servair: keep crawlers out of layered navigation (added 2026-08-26) ---
# Faceted filter URLs are infinite permutations; each is uncacheable and runs a
# full Elasticsuite query. A distributed flood against these took the site down
# on 2026-08-26. These are also duplicate content and should never be indexed.
Disallow: /*?*brand=
Disallow: /*?*brand[
Disallow: /*?*product_type_1=
Disallow: /*?*product_type_1[
Disallow: /*?*is_ajax=
Disallow: /*?*price=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_dir=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_mode=

# --- Servair: AI agents / LLM crawlers (added 2026-08-31) ---
# Deliberately NOT declaring per-bot groups (GPTBot, ClaudeBot, PerplexityBot, ...).
# A named User-agent group REPLACES the "User-agent: *" group for that bot, which would
# silently exempt it from the layered-navigation Disallow rules above — the exact surface
# that took the site down on 2026-08-26. The "*" group already allows every AI crawler
# everything it is not explicitly denied, so per-bot Allow groups add nothing but risk.
#
# Agents: read /llms.txt first, then /products.json for the full catalogue.
# Both are plain files at the document root and are not covered by "Disallow: /pub/".
Allow: /llms.txt
Allow: /products.json
Allow: /google_merchant_feed.csv

# --- Servair: remaining layered-navigation attributes (added 2026-09-01) ---
# The 2026-08-26 block covered brand / product_type_1 / price only. Googlebot was still free to
# crawl the other 8 filterable attributes, hit the pub/index.php flood guard, and collect 403s:
# 911 pages in Search Console under "Blocked due to access forbidden (403)", every sampled example
# a size[] permutation such as /parts.html?size[0]=5+Ton&size[1]=20+Ton&...
# Blocking them here turns those errors into an intentional "blocked by robots.txt" state and stops
# the wasted crawl budget. Source of the list: attributes with is_filterable=1 in catalog_eav_attribute.
Disallow: /*?*size=
Disallow: /*?*size[
Disallow: /*?*type=
Disallow: /*?*type[
Disallow: /*?*color=
Disallow: /*?*color[
Disallow: /*?*frequency=
Disallow: /*?*frequency[
Disallow: /*?*horsepower=
Disallow: /*?*horsepower[
Disallow: /*?*manufacturer=
Disallow: /*?*manufacturer[
Disallow: /*?*mgs_brand=
Disallow: /*?*mgs_brand[
Disallow: /*?*bitzer_subcategory=
Disallow: /*?*bitzer_subcategory[