83 lines
1.3 KiB
Plaintext
83 lines
1.3 KiB
Plaintext
|
# thank you to starbreaker for making his robots.txt
|
||
|
|
# and ai.txt files available for others to use
|
||
|
|
|
||
|
|
# Want my work for your dataset? Fuck you, pay me.
|
||
|
|
|
||
|
|
User-Agent: *
|
||
|
|
|
||
|
|
# Text Permissions
|
||
|
|
Disallow: *.txt
|
||
|
|
Disallow: *.md
|
||
|
|
Disallow: *.markdown
|
||
|
|
Disallow: *.org
|
||
|
|
Disallow: *.rst
|
||
|
|
Disallow: *.pdf
|
||
|
|
Disallow: *.doc
|
||
|
|
Disallow: *.docx
|
||
|
|
Disallow: *.odt
|
||
|
|
Disallow: *.rtf
|
||
|
|
Disallow: *.tex
|
||
|
|
Disallow: *.wks
|
||
|
|
Disallow: *.wpd
|
||
|
|
Disallow: *.wps
|
||
|
|
Disallow: *.htm
|
||
|
|
Disallow: *.html
|
||
|
|
Disallow: *.shtml
|
||
|
|
Disallow: *.xml
|
||
|
|
Disallow: *.rss
|
||
|
|
|
||
|
|
# Images Permissions
|
||
|
|
Disallow: *.bmp
|
||
|
|
Disallow: *.gif
|
||
|
|
Disallow: *.ico
|
||
|
|
Disallow: *.jpeg
|
||
|
|
Disallow: *.jpg
|
||
|
|
Disallow: *.png
|
||
|
|
Disallow: *.svg
|
||
|
|
Disallow: *.tif
|
||
|
|
Disallow: *.tiff
|
||
|
|
Disallow: *.webp
|
||
|
|
|
||
|
|
# Audio Permissions
|
||
|
|
Disallow: *.aac
|
||
|
|
Disallow: *.aiff
|
||
|
|
Disallow: *.amr
|
||
|
|
Disallow: *.flac
|
||
|
|
Disallow: *.m4a
|
||
|
|
Disallow: *.mp3
|
||
|
|
Disallow: *.oga
|
||
|
|
Disallow: *.opus
|
||
|
|
Disallow: *.wav
|
||
|
|
Disallow: *.wma
|
||
|
|
|
||
|
|
# Video Permissions
|
||
|
|
Disallow: *.mp4
|
||
|
|
Disallow: *.webm
|
||
|
|
Disallow: *.ogg
|
||
|
|
Disallow: *.avi
|
||
|
|
Disallow: *.mov
|
||
|
|
Disallow: *.wmv
|
||
|
|
Disallow: *.flv
|
||
|
|
Disallow: *.mkv
|
||
|
|
|
||
|
|
# Code Permissions
|
||
|
|
Disallow: *.py
|
||
|
|
Disallow: *.js
|
||
|
|
Disallow: *.java
|
||
|
|
Disallow: *.c
|
||
|
|
Disallow: *.cpp
|
||
|
|
Disallow: *.cs
|
||
|
|
Disallow: *.h
|
||
|
|
Disallow: *.css
|
||
|
|
Disallow: *.php
|
||
|
|
Disallow: *.swift
|
||
|
|
Disallow: *.go
|
||
|
|
Disallow: *.rb
|
||
|
|
Disallow: *.pl
|
||
|
|
Disallow: *.sh
|
||
|
|
Disallow: *.sql
|
||
|
|
Disallow: *.xsl
|
||
|
|
|
||
|
|
# Disallow
|
||
|
|
Disallow: /
|