From 04f32b20fae8795aab0c1f8b703394056f3efea3 Mon Sep 17 00:00:00 2001
From: Adam Dickmeiss <adam@indexdata.dk>
Date: Fri, 8 Dec 2000 22:46:53 +0000
Subject: [PATCH] File robots.txt now read the each domain. Pages are now
 fetched in a Round-robin fashion.

---
 dcdot.tcl |    6 ++++--
 robot.tcl |   72 ++++++++++++++++++++++++++++++++++++++++---------------------
 2 files changed, 52 insertions(+), 26 deletions(-)

diff --git a/dcdot.tcl b/dcdot.tcl
index af0bef7..a29e6f5 100755
--- a/dcdot.tcl
+++ b/dcdot.tcl
@@ -1,5 +1,5 @@
 #!/usr/bin/tclsh 
-# $Id: dcdot.tcl,v 1.2 2000/12/08 08:55:35 adam Exp $
+# $Id: dcdot.tcl,v 1.3 2000/12/08 22:46:53 adam Exp $
 #
 
 proc RobotRestart {} {
@@ -99,7 +99,6 @@ proc RobotReadHeader {url sock} {
 		    }
 		}
 		default {
-		    Robot404 $url
 		    close $sock
 		    RobotRestart
 		}
@@ -170,4 +169,7 @@ if {$argc == 1} {
 	    puts $m
         }
     }
+    foreach v [array names URL $url,head,*] {
+	puts "$v = $URL($v)"
+    }
 }
diff --git a/robot.tcl b/robot.tcl
index 93c4541..15468bf 100755
--- a/robot.tcl
+++ b/robot.tcl
@@ -1,7 +1,7 @@
 #!/usr/bin/tclsh 
-# $Id: robot.tcl,v 1.6 2000/12/07 20:16:11 adam Exp $
+# $Id: robot.tcl,v 1.7 2000/12/08 22:46:53 adam Exp $
 #
-proc RobotFileNext {area} {
+proc RobotFileNext1 {area} {
     if {[catch {set ns [glob ${area}/*]}]} {
         return {}
     }
@@ -18,7 +18,7 @@ proc RobotFileNext {area} {
     }
     foreach n $ns {
 	if {[file isdirectory $n]} {
-            set sb [RobotFileNext $n]
+            set sb [RobotFileNext1 $n]
             if {[string length $sb]} {
                 return $sb
             }
@@ -27,6 +27,45 @@ proc RobotFileNext {area} {
     return {}
 }
 
+proc RobotFileWait {} {
+    global robotSeq
+    set robotSeq 0
+}
+
+proc RobotFileNext {area} {
+    global robotSeq
+    if {[catch {set ns [glob ${area}/*]}]} {
+        return {}
+    }
+    set off [string length $area]
+    incr off
+
+    set n [lindex $ns $robotSeq]
+    if {![string length $n]} {
+	puts "------------ N E X T  R O U N D --------"
+	set robotSeq -1
+	after 2000 RobotFileWait
+	vwait robotSeq
+
+	set n [lindex $ns $robotSeq]
+	if {![string length $n]} {
+	    return {}
+	}
+    }
+    incr robotSeq
+    if {[file isfile $n/robots.txt]} {
+	puts "ok returning http://[string range $n $off end]/robots.txt"
+	return http://[string range $n $off end]/robots.txt
+    } elseif {[file isdirectory $n]} {
+	set sb [RobotFileNext1 $n]
+	if {[string length $sb]} {
+	    return $sb
+	}
+    }
+    return {}
+}
+
+
 proc RobotFileExist {area host path} {
     set comp [split $area/$host$path /]
     set l [llength $comp]
@@ -84,6 +123,11 @@ proc RobotFileOpen {area host path {mode w}} {
         if {[catch {cd ./$d}]} {
             exec mkdir $d
             cd ./$d
+	    if {![string compare $area unvisited] && $i == 1 && $mode == "w"} {
+		set out [open robots.txt w]
+		puts "creating robots.txt in $d"
+		close $out
+	    }
         }
     }
     set d [lindex $comp $len]
@@ -541,25 +585,8 @@ proc bgerror {m} {
     puts "BGERROR $m"
 }
 
-if {0} {
-    proc RobotRestart {} {
-        global robotMoreWork
-        set robotMoreWork 0
-	puts "myrestart"
-    }
-    set robotMoreWork 1
-    set url {http://www.indexdata.dk/zap/}
-    RobotGetUrl $url {}
-    while {$robotMoreWork} {
-	vwait robotMoreWork
-    }
-    puts "-----------"
-    puts $URL($url,buf)
-    puts "-----------"
-    exit 1
-}
-
 set robotMoreWork 0
+set robotSeq 0
 set workdir [pwd]
 
 if {[llength $argv] < 2} {
@@ -575,9 +602,6 @@ if {[string length $site]} {
     if [RobotGetUrl $site {}] {
 	set robotMoreWork 0
 	puts "Couldn't process $site"
-    } else {
-	#set x [RobotFileOpen unvisited $site /robots.txt]
-	#RobotFileClose $x
     }
 }
 
-- 
1.7.10.4