syntax robotstxt

list -i field user-agent: disallow: allow: crawl-delay: sitemap: host:

state main code
    char -b A-Za-z firstword
    char # comment
    char " \t" this
    noeat remainder

state firstword
    char -b A-Za-z- this
    char -b : field
    noeat -b remainder

state field
    inlist field value keyword
    recolor error
    noeat remainder

state value
    char # comment
    char "\n" main
    eat this

state remainder error
    recolor error
    char # comment
    char "\n" main
    eat this

state comment
    char "\n" main
    eat this
