# General catchall rules
User-agent: *
Allow: /$
Allow: /?
Allow: /flights/
Allow: /flights?
Allow: /flights$
Allow: /cheap-flights
Allow: /email-deals
Allow: /fr/cheap-flights
Allow: /fr/email-deals
Allow: /cf-assets/images/
Allow: /cf-assets
Allow: /airports

## Common Paths
Disallow: /font-awesome
Disallow: /bootstrap
Disallow: /dispute
Disallow: /self-transfer
Disallow: /cf-fonts
Disallow: /css
Disallow: /fonts
Disallow: /gass
Disallow: /nuyp
Disallow: /js
Disallow: /storefront-api
Disallow: /material-icomoon
Disallow: /mobile-flags
Disallow: /upsell
Disallow: /seatmap
Disallow: /surveys
Disallow: /flight-search-api/
Disallow: /autocomplete
Disallow: /storefront-api/
Disallow: /gass/
Disallow: /cdn-cgi/

## English Paths
Disallow: /fare-alerts
Disallow: /privacy-policy
Disallow: /privacypolicy
Disallow: /terms-and-conditions
Disallow: /fares
Disallow: /account
Disallow: /my-account
Disallow: /checkout
Disallow: /booking
Disallow: /support
Disallow: /hotel
Disallow: /flight
Disallow: /cars
Disallow: /hotels
Disallow: /service
Disallow: /landing-flights/ajax

## French Paths (same order as English)
Disallow: /fr/fare-alerts
Disallow: /fr/privacy-policy
Disallow: /fr/privacypolicy
Disallow: /fr/terms-and-conditions
Disallow: /fr/fares
Disallow: /fr/account
Disallow: /fr/my-account
Disallow: /fr/checkout
Disallow: /fr/booking
Disallow: /fr/support
Disallow: /fr/hotel
Disallow: /fr/flight
Disallow: /fr/cars
Disallow: /fr/hotels
Disallow: /fr/service
Disallow: /fr/landing-flights/ajax




# Advertiser Group: Paid Advertising Crawlers
## Google Ads bots
User-agent: Google-HotelAdsVerifier
User-agent: AdsBot-Google-Mobile-Apps
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google

## Meta Ads bots
User-agent: meta-externalads
User-agent: meta-externalagent
User-agent: meta-externalfetcher

## Criteo Ads bot
User-agent: CriteoBot

## Microsoft Ads bot
User-agent: AdIdxBot
Allow: /$
Allow: /?
Allow: /cheap-flights
Allow: /email-deals
Allow: /fr/cheap-flights
Allow: /fr/email-deals
Allow: /cf-assets/images/
Disallow: /




# Advertiser: Google AdSense - Requires broad access for ad display validation
User-agent: Mediapartners-Google
Disallow:




# Unfavorable Bots
# SemrushBot - Semrush Inc. - Will hammer the site endlessly for competition intel
User-agent: SemrushBot
# EyeMonIT - EyeMonIT - Constant health checks that bury your server
User-agent: EyeMonIT
# Buck - Buck - Consumes resources for shady marketing data
User-agent: Buck
# Mediatoolkitbot - Mediatoolkit - Scrapes content non-stop for social metrics
User-agent: Mediatoolkitbot
# SeznamBot - Seznam.cz - Floods pages for Czech search, slowing global users
User-agent: SeznamBot
# Taboolabot - Taboola - Sucks bandwidth to serve more native ads elsewhere
User-agent: Taboolabot
# Mail.Ru - Mail.ru - Heavy Russian crawler draining crawl budget
User-agent: Mail.Ru
# bidswitchbot - BidSwitch - Harvests pages for ad bidding profiles around the clock
User-agent: bidswitchbot
# MJ12bot - Majestic - Builds backlink maps while saturating your logs
User-agent: MJ12bot
# ZoomBot - Unknown - Aggressive spider that chokes small servers
User-agent: ZoomBot
# seostar - SEOstar - Obsessive SEO crawler that doesn’t respect rate limits
User-agent: seostar
# Neevabot - Neeva - Crawls relentlessly even though search engine is defunct
User-agent: Neevabot
# ZoominfoBot - ZoomInfo - Mines personal data for B2B dossiers 24/7
User-agent: ZoominfoBot
# Linespider - Unknown - Spams requests causing performance hits
User-agent: Linespider
# AhrefsBot - Ahrefs - Eats up bandwidth mapping links for competitors
User-agent: AhrefsBot
# DotBot - Moz - Crawls full site to fuel Moz index at massive scale
User-agent: DotBot
# BLEXBot - Blex - Extracts links quickly causing spike in CPU
User-agent: BLEXBot
# MegaIndex.ru - MegaIndex - Russian SEO crawler notorious for high request rates
User-agent: MegaIndex.ru
# MegaIndexBot - MegaIndex - Duplicate aggressive crawler variant
User-agent: MegaIndexBot
# SEOkicks-Robot - SEOkicks - German SEO bot flooding pages for link data
User-agent: SEOkicks-Robot
# spbot - Seo-Profiler - Rapid-fire requests that exhaust resources
User-agent: spbot
# Barkrowler - Common Crawl - Full-site archive sweeps causing heavy load
User-agent: Barkrowler
# PetalBot - Huawei - Huge crawler building Petal search index globally
User-agent: PetalBot
# Bytespider - ByteDance - TikTok parent collects data aggressively
User-agent: Bytespider
# MauiBot - Unknown - Exotic bot that blitzes pages without mercy
User-agent: MauiBot
# Baiduspider - Baidu - Massive Chinese crawler that overloads sites
User-agent: Baiduspider
# Sogou Spider - Sogou - Another Chinese search bot taxing bandwidth
User-agent: Sogou Spider
# YandexBot - Yandex - Russian crawler known for deep recursive fetches
User-agent: YandexBot
# CCBot - Common Crawl - Huge web archiving bot devouring crawl budget
User-agent: CCBot
disallow: /