Create a custom robots.txt file for your website. Configure crawler access, add sitemap URLs, preview your rules, and download your file in seconds.
Configure your rules
Choose the paths search crawlers should be allowed or disallowed to request.
All crawlers (*)
Googlebot
Bingbot
Googlebot-Image
GPTBot
Custom crawler
PathAction
Support varies by crawler. Invalid directives will not be accepted.
Generated robots.txt
Updates automatically as you edit.
robots.txt
Before publishing: Replace example sitemap URLs with your real sitemap URL. Review each rule carefully. A disallow rule can prevent crawling of important pages, and robots.txt is not a security or privacy mechanism.
(function(){
const root=document.getElementById('db-robots-tool');
if(!root||root.dataset.ready==='1')return;
root.dataset.ready='1';
const $=id=>root.querySelector('#'+id);
const rules=$('db-rt-rules');
const output=$('db-rt-output');
const warning=$('db-rt-warning');
let ruleId=0;function addRule(path='',action='Disallow'){
const row=document.createElement('div');
row.className='rt-rule';
row.dataset.ruleId=String(++ruleId);
const input=document.createElement('input');
input.type='text';
input.placeholder='/private/';
input.value=path;
input.setAttribute('aria-label','Path');
const select=document.createElement('select');
select.setAttribute('aria-label','Rule action');
[['Disallow','Disallow'],['Allow','Allow']].forEach(([value,label])=>{
const option=document.createElement('option');
option.value=value;option.textContent=label;select.appendChild(option);
});
select.value=action;
const remove=document.createElement('button');
remove.type='button';remove.className='rt-remove';
remove.textContent='×';remove.setAttribute('aria-label','Remove rule');
remove.addEventListener('click',()=>{row.remove();generate();});
input.addEventListener('input',generate);
select.addEventListener('change',generate);
row.append(input,select,remove);
rules.appendChild(row);
}function safePath(value){
const path=value.trim();
if(!path)return '';
if(!path.startsWith('/'))return null;
if(/[\r\n#]/.test(path))return null;
return path;
}function generate(){
const warnings=[];
const bot=$('db-rt-bot').value;
const agent=bot==='Custom'?$ ('db-rt-custom').value.trim():bot;
if(!agent||/[\r\n#\s]/.test(agent)){
warnings.push('Enter a valid custom crawler user-agent without spaces or line breaks.');
}const lines=['User-agent: '+(agent||'*')];
if($('db-rt-admin').checked)lines.push('Disallow: /wp-admin/');
if($('db-rt-ajax').checked)lines.push('Allow: /wp-admin/admin-ajax.php');
if($('db-rt-search').checked)lines.push('Disallow: /*?s=');rules.querySelectorAll('.rt-rule').forEach(row=>{
const path=safePath(row.querySelector('input').value);
if(path===null){
if(row.querySelector('input').value.trim())warnings.push('Each custom path must start with / and cannot contain line breaks or #.');
return;
}
if(path)lines.push(row.querySelector('select').value+': '+path);
});const sitemapText=$('db-rt-sitemap').value.trim();
const sitemaps=[];
if(sitemapText){
sitemapText.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).forEach(url=>{
try{
const parsed=new URL(url);
if(!['https:','http:'].includes(parsed.protocol)||parsed.username||parsed.password){
warnings.push('A sitemap URL is invalid. Use a full public HTTP or HTTPS URL.');
}else if(parsed.hash||/[\r\n]/.test(url)){
warnings.push('Sitemap URLs must not contain fragments or line breaks.');
}else{
sitemaps.push(parsed.href);
}
}catch(e){warnings.push('A sitemap URL is invalid. Include the full URL, such as https://example.com/sitemap.xml.');}
});
}const extra=$('db-rt-extra').value.trim();
const extraLines=[];
if(extra){
extra.split(/\r?\n/).forEach(line=>{
const trimmed=line.trim();
if(!trimmed)return;
if(/[\r\n]/.test(trimmed)||!/^(Crawl-delay|Host):\s*\S.+$/i.test(trimmed)){
warnings.push('An additional directive was skipped. Only basic Crawl-delay and Host directives are accepted here.');
return;
}
extraLines.push(trimmed);
});
}let text=lines.join('\n');
if(extraLines.length)text+='\n'+extraLines.join('\n');
if(sitemaps.length)text+='\n\n'+Array.from(new Set(sitemaps)).map(url=>'Sitemap: '+url).join('\n');
output.value=text+'\n';if(agent&&agent!=='*'&&agent!=='Googlebot'&&agent!=='Bingbot'&&agent!=='Googlebot-Image'&&agent!=='GPTBot'){
warnings.push('Confirm that this crawler uses the exact user-agent you entered.');
}
if(agent==='GPTBot')warnings.push('GPTBot is a crawler operated by OpenAI. These rules do not automatically control every AI crawler or service.');
if(agent!=='*'&&agent)warnings.push('You selected a named crawler. Rules for other crawlers are not included. Use All crawlers if that is your intention.');
warning.textContent=Array.from(new Set(warnings)).join(' ');
warning.classList.toggle('visible',warnings.length>0);
}$('db-rt-bot').addEventListener('change',()=>{
$('db-rt-custom-wrap').hidden=$('db-rt-bot').value!=='Custom';
generate();
});
['db-rt-custom','db-rt-admin','db-rt-ajax','db-rt-search','db-rt-sitemap','db-rt-extra'].forEach(id=>{
$(id).addEventListener('input',generate);
$(id).addEventListener('change',generate);
});
$('db-rt-add').addEventListener('click',()=>{addRule();generate();});
$('db-rt-reset').addEventListener('click',()=>{
$('db-rt-bot').value='*';
$('db-rt-custom').value='';
$('db-rt-custom-wrap').hidden=true;
$('db-rt-admin').checked=true;
$('db-rt-ajax').checked=true;
$('db-rt-search').checked=false;
$('db-rt-sitemap').value='';
$('db-rt-extra').value='';
rules.replaceChildren();
addRule();
generate();
});$('db-rt-copy').addEventListener('click',async()=>{
try{
await navigator.clipboard.writeText(output.value);
$('db-rt-copy').textContent='Copied!';
setTimeout(()=>$('db-rt-copy').textContent='Copy contents',1600);
}catch(e){
output.readOnly=false;output.focus();output.select();
const ok=document.execCommand('copy');
output.readOnly=true;
$('db-rt-copy').textContent=ok?'Copied!':'Select and copy';
setTimeout(()=>$('db-rt-copy').textContent='Copy contents',1600);
}
});$('db-rt-download').addEventListener('click',()=>{
const blob=new Blob([output.value],{type:'text/plain;charset=utf-8'});
const url=URL.createObjectURL(blob);
const link=document.createElement('a');
link.href=url;link.download='robots.txt';
document.body.appendChild(link);link.click();link.remove();
URL.revokeObjectURL(url);
});addRule();
generate();
})();
