#VERSION,2.01
# $Id: nikto_robots.plugin 152 2009-08-13 19:37:44Z deity $

###############################################################################
#  Copyright (C) 2004 CIRT, Inc.
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; version 2
#  of the License only.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################

###############################################################################
# PURPOSE
# Check out the robots.txt file
###############################################################################
sub nikto_robots_init
{
   my $id =
   {
      name         => "robots",
      full_name    => "Robots",
      author       => "Sullo",
      description  => "Checks whether there's anything within the robots.txt file and analyses it for other paths to pass to other scripts.",
      recon_method => \&nikto_robots,
      recon_weight => 49,
      copyright    => "2008 CIRT Inc."
   };
   return $id;
}

sub nikto_robots
{
   my ($mark) = @_;
   (my $RES, $CONTENT) = nfetch($mark,"/robots.txt", "GET");

   if (($RES eq 200) || ($RES eq $FoF{okay}{response}))    # got one!
   {
      if (is_404("robots.txt", $CONTENT, $RES)) { return; }

      my ($DIRS, $RFILES) = "";
      my $DISCTR = 0;
      my @DOC = split(/\n/, $CONTENT);
      foreach my $line (@DOC)
      {
         $line = char_escape($line);
         if ($line =~ /allow/i)                       # hmmm...
         {
            chomp($line);
            $line =~ s/\#.*$//;
            $line =~ s/(^\s+|\s+$)//g;
            $line =~ s/\s+/ /g;
            $line =~ s/\\t/ /g;
            $line =~ s/(dis)?allow(\\:)?( )?//i;
            $line =~ s/\*//g;
            $line =~ s/\/+/\//g;

            if ($line eq "") { next; }
   
            # try to figure out file vs dir... just guess...
            if (($line !~ /\./) && ($line !~ /\/$/)) { $line .= "/"; }

            $line = LW2::uri_normalize($line);

            # figure out dirs/files...
            my $realdir  = LW2::uri_get_dir($line);
            my $realfile = $line;
            $realfile =~ s/^$realdir//;

            nprint("- robots.txt entry dir:$realdir -- file:$realfile", "d");
            if (($realdir  ne "") && ($realdir  ne "/")) { $DIRS{$realdir}++; }
            if (($realfile ne "") && ($realfile ne "/")) { $RFILES{$realfile}++; }
            $DISCTR++;
         }    # end if $line =~ disallow
      }    # end foreach my $line (@DOC)_

      # add them  to mutate dir/file
      my $raw;
      foreach $raw (sort keys %DIRS)
      {
         $raw =~ s/\\//g;
         if ($VARIABLES{"\@MUTATEDIRS"} !~ /$raw/)
         {
            $VARIABLES{"\@MUTATEDIRS"} .= " $raw";
         }
         if ($raw =~ /cgi/ && $VARIABLES{"\@CGIDIRS"} !~ /$raw/)
         {
            $VARIABLES{"\@CGIDIRS"} .= " $raw";
         }
         if ($raw =~ /forum/ && $VARIABLES{"\@NUKE"} !~ /$raw/)
         {
            $VARIABLES{"\@NUKE"} .= " $raw";
         }
         if ($raw =~ /pass/ && $VARIABLES{"\@PASSWORDDIRS"} !~ /$raw/)
         {
            $VARIABLES{"\@PASSWORDDIRS"} .= " $raw";
         }
      }

      foreach $raw (sort keys %RFILES)
      {
         $raw =~ s/\\//g;
         if ($VARIABLES{"\@MUTATEFILES"} !~ /$raw/)
         {
            $VARIABLES{"\@MUTATEFILES"} .= " $raw";
         }
         if ($raw =~ /pass/ && $VARIABLES{"\@PASSWORDFILES"} !~ /$raw/)
         {
            $VARIABLES{"\@PASSWORDFILES"} .= " $raw";
         }
      }

      my $msg;
      if ($DISCTR eq 1)   { $msg = "contains $DISCTR entry which should be manually viewed."; }
      elsif ($DISCTR > 1) { $msg = "contains $DISCTR entries which should be manually viewed."; }
      else                { $msg = "retrieved but it does not contain any 'disallow' entries (which is odd)."; }

      add_vulnerability($mark,"robots.txt $msg", 999996, 0, "GET", "/robots.txt",\%result);
   }
}

1;