[divide-and-conquer~concur-next:4] State of play as of Dec. 9th.
- From: TimBray@kenai.com
- To: commits@divide-and-conquer.kenai.com
- Subject: [divide-and-conquer~concur-next:4] State of play as of Dec. 9th.
- Date: Thu, 10 Dec 2009 01:02:27 +0000
Project: divide-and-conquer
Repository: concur-next
Revision: 4
Author: TimBray
Date: 2009-12-10 01:02:25 UTC
Link:
Log Message:
------------
State of play as of Dec. 9th.
Revisions:
----------
4
Modified Paths:
---------------
src/org/tbray/paralines.clj
src/org/tbray/overhead.clj
src/org/tbray/JOverhead.java
src/org/tbray/popular.clj
src/org/tbray/ref_based.clj
src/org/tbray/pop_agent.clj
Added Paths:
------------
src/org/tbray/read_lines.clj
src/org/tbray/wf2.clj
Diffs:
------
Index: src/org/tbray/ref_based.clj
===================================================================
--- src/org/tbray/ref_based.clj (revision 3)
+++ src/org/tbray/ref_based.clj (revision 4)
@@ -1,4 +1,3 @@
-
(ns org.tbray.ref-based
;(:require )
;(:use )
Index: src/org/tbray/pop_agent.clj
===================================================================
--- src/org/tbray/pop_agent.clj (revision 3)
+++ src/org/tbray/pop_agent.clj (revision 4)
@@ -1,7 +1,5 @@
+(ns org.tbray.pop-agent)
-(ns org.tbray.pop-agent
- )
-
(def re #"GET /ongoing/When/\d\d\dx/(\d\d\d\d/\d\d/\d\d/[^ .]+) ")
(defn add [so-far target]
Index: src/org/tbray/overhead.clj
===================================================================
--- src/org/tbray/overhead.clj (revision 3)
+++ src/org/tbray/overhead.clj (revision 4)
@@ -5,24 +5,22 @@
(def READ-ONLY-MODE (. java.nio.channels.FileChannel$MapMode READ_ONLY))
-(defn blok [ channel offset size ]
- (let [buffer (. channel map READ-ONLY-MODE offset size)
- decoder (. (. java.nio.charset.Charset forName "UTF-8") newDecoder)
- chars (. decoder decode buffer)
- text (. chars toString)
- tail (. text substring (- (. text length) 5))]
+(defn blok [ in size ]
+ (let [buffer (make-array Byte/TYPE size)
+ bytes (. in read buffer)]
(println (str "T=" tail))))
(defn blox [file-name settings]
(let [block-size (or (settings :block-size) (* 32 1024 1024))
- channel (. (new FileInputStream file-name) getChannel)
- data-size (. channel size)
+ file (. new File file-name)
+ in (new FileInputStream file-name)
+ data-size (. file length)
block-count (quot data-size block-size)
tail-size (mod data-size block-size) ]
(loop [index 0]
(if (< index block-count)
(do
- (blok channel (* index block-size) block-size)
+ (blok in block-size)
(recur (inc index)))))
- (blok channel (* block-count block-size) tail-size)
+ (blok in tail-size)
(println (str "DONE: " block-count " blocks."))))
Index: src/org/tbray/paralines.clj
===================================================================
--- src/org/tbray/paralines.clj (revision 3)
+++ src/org/tbray/paralines.clj (revision 4)
@@ -1,42 +1,52 @@
-
(ns org.tbray.paralines
(:import (java.io FileInputStream)))
(def READ-ONLY-MODE (. java.nio.channels.FileChannel$MapMode READ_ONLY))
-(defn patcher
- "fragments is a map whose keys are of the form (str index :front|:back)
telling you
-it's a partial line off the front or back of the block."
+(defn frag
+ "fragments is a map whose keys are of the form (str index :front|:back)"
[fragments fragment index front-back]
- (assoc fragments (str index front-back) (new String fragment)))
+ (dosync
+ (ref-set fragments (assoc @fragments (str index front-back) (new String
fragment)))))
+
+(defn join-fragments [fragments-in params]
+ (loop [fragments fragments-in accum nil]
+ (if (not (empty? fragments))
+ (let [[key frag] (first fragments)
+ [istring fb] (. #":" split key)
+ index (new Integer istring)
+ is-front (= fb "front")
+ other-key (if is-front (str (dec index) ":back") (str (inc
index) ":front"))
+ other (fragments other-key)
+ line (if is-front (str other frag) (str frag other))
+ next-accum ((params :per-line) accum line) ]
+ (recur (dissoc fragments key other-key) next-accum))
+ ((params :done-lines) accum))))
(defn read-block
"Read a block at the given index from the I/O channel, divide it up into
lines and send them off to the destination. Also handle partials lines from
the
beginning and ends of blocks."
[ b accumulator ] ; b short for block since we're using it a lot
- (println (str "Block " (b :index)))
+ (if (= 0 (mod (b :index) 1000))
+ (println (str "Block " (b :index))))
(let [buffer (. (b :channel) map READ-ONLY-MODE (b :offset) (b :size))
decoder (. (. java.nio.charset.Charset forName "UTF-8") newDecoder)
- chars (. decoder decode buffer)
- text (. chars toString)
- first-nl (. text indexOf 10)
- front (. text substring 0 first-nl)
- last-nl (. text lastIndexOf 10)
- back (. text substring (inc last-nl))
- fragments (b :fragments) index (b :index) per-line (b :per-line)]
+ text (. (. decoder decode buffer) toString)
+ index (b :index)
+ trailing-nl (. text endsWith "\n")
+ chunks (. #"\n" split text)
+ front (first chunks)
+ lc (get chunks (dec (count chunks))) ; (last chunks) tickles a
Clojure 1.0 bug
+ back (if trailing-nl "" lc)
+ lines (if trailing-nl (rest chunks) (butlast (rest chunks)))
+ accum (reduce (b :per-line) accumulator lines) ] ; all the work
happens here
- (send fragments patcher front index :front)
- (if (not (b :last))
- (send fragments patcher back index :back))
- (await fragments)
+ (frag (b :fragments) front index :front) ; save front/back fragments
+ (if (not last)
+ (frag (b :fragments) back index :back))
+ accum))
- (loop [ nl first-nl accum accumulator ]
- (let [ from (inc nl) to (. text indexOf 10 from) ]
- (if (< from last-nl)
- (recur to (per-line (new String (. text substring from to)) accum))
- accum)))))
-
(defn next-block [params index]
(let [next (dosync (alter index inc))
this (dec next)
@@ -51,7 +61,7 @@
(loop [ accumulator nil block (next-block params index) ]
(if block
(recur (read-block block accumulator) (next-block params index))
- ((params :per-line) nil accumulator))))
+ ((params :done-lines) accumulator))))
; There's probably a map/doseq/dorun one-liner that'll do this, sigh
(defn make-threads [count params index]
@@ -70,9 +80,9 @@
other-key (if is-front (str (dec index) ":back") (str (inc
index) ":front"))
other (fragments other-key)
line (if is-front (str other frag) (str frag other))
- next-accum ((params :per-line) line accum) ]
+ next-accum ((params :per-line) accum line)]
(recur (dissoc fragments key other-key) next-accum))
- ((params :per-line) nil accum))))
+ ((params :done-lines) accum))))
(defn read-lines
"Reads the named file and feeds the lines one at a time to the function
@@ -84,9 +94,9 @@
value for the first (line) argument, that signals the final call in some
given
thread, so that the function may choose to do wind-up and reporting work.
-When all the lines have been processed, the the 'all-done' function will be
+When all the lines have been processed, the the 'all-done' function will be
called with 'user-data' as its single argument. The 'destination' function
-will be called on multiple threads in parallel, so its logic must be
+will be called on multiple threads in parallel, so its logic must be
thread-safe. There is no guarantee in which order the lines from the file
will be processed. The 'settings' argument is a map; the default values for
the
number of threads and file block-size may be overridden with the :width and
@@ -95,21 +105,19 @@
(let [block-size (or (settings :block-size) (* 10 1024 1024))
channel (. (new FileInputStream file-name) getChannel)
data-size (. channel size)
- fragments (agent {})
+ fragments (ref {})
block-count (quot data-size block-size)
thread-count (max 1 (min block-count (or (settings :width) 10)))
tail-size (mod data-size block-size)
- params {:block-count block-count
- :block-size block-size
- :tail-size tail-size
+ index (ref 0)
+ params {:block-count block-count :block-size block-size :tail-size
tail-size
:channel channel
- :per-line (fn [line accum] (destination line user-data
accum))
+ :per-line (fn [accum #^String line] (destination (new String
line) user-data accum))
+ :done-lines (fn [accum] (destination nil user-data accum))
:fragments fragments }
- index (ref 0)
threads (make-threads thread-count params index)]
- ;; prime the first/last fragments pump
- (send fragments patcher "" -1 :back)
+ (frag fragments "" -1 :back) ; prime the fragments pump
(doseq [thread threads]
(.start thread)
@@ -117,6 +125,5 @@
(doseq [thread threads]
(.join thread))
- (await fragments)
- (join-fragments @fragments params)
+ (join-fragments (deref fragments) params)
(all-done user-data)))
\ No newline at end of file
Index: src/org/tbray/wf2.clj
===================================================================
--- src/org/tbray/wf2.clj (revision 0)
+++ src/org/tbray/wf2.clj (revision 4)
@@ -0,0 +1,180 @@
+(ns org.tbray.wf2
+ ;(:use clojure.contrib.test-is) ; having trouble getting this working on
the production machine
+ )
+
+(def re #"^/ongoing/When/\d\d\dx/\d\d\d\d/\d\d/\d\d/[^ .]+$")
+(defn new-accum [] { :hits {} :bytes {} :404s {} :fetchers {} :referers {}
})
+
+(defn bump [accum key uri increment]
+ (let [totals (accum key)
+ currently (if-let [total (totals uri)] total 0)]
+ (assoc accum key (assoc totals uri (+ currently increment)))))
+
+(defn merger [current incoming]
+ (loop [keys (keys current) output {}]
+ (if-let [key (first keys)]
+ (let [merge-1 (merge-with + (current key) (incoming key))]
+ (recur (rest keys) (assoc output key merge-1)))
+ output)))
+
+(defn proc-line [line so-far accum-1]
+ (if (nil? line)
+ (send so-far merger accum-1)
+ (let [accum-2 (if accum-1 accum-1 (new-accum))
+ fields (. #" " split line)
+ ; [client _ _ _ _ _ uri _ bstring status ref] (. #" " split line)
+ #^String uri-1 (aget fields 6) #^String uri (. uri-1 intern)
+ #^String status (aget fields 8)
+ accum-3 (if (= status "404") (bump accum-2 :404s uri 1) accum-2)
+ #^String bstring (aget fields 9)
+ accum-4 (if (re-find #"^\d+$" bstring)
+ (bump accum-3 :bytes uri (new Integer bstring))
+ accum-3)]
+ (if (re-find re uri)
+ (let [accum-5 (bump accum-4 :hits uri 1)
+ #^String ref (aget fields 10)
+ accum-6 (if (or (= ref "\"-\"") (re-find #"tbray.org/" ref))
+ accum-5 (bump accum-5 :referers ref 1))
+ #^String client (aget fields 0)
+ accum-7 (bump accum-6 :fetchers client 1) ]
+ accum-7)
+ accum-4))))
+
+(defn top-10 [ hash ]
+ (take 10 (sort-by last (fn [a b] (> a b)) hash)))
+
+(defn print-report [stats-in]
+ (loop [stats stats-in]
+ (when-let [stat (first stats)]
+ (println (str "Top " (first stat)))
+ (println (str " ..." (first (last stat)) "... "))
+ (recur (rest stats)))))
+
+(defn report-wf2 [accum-ref]
+ (loop [accum @accum-ref stats {}]
+ (if-let [stat (first accum)]
+ (recur (rest accum) (assoc stats (first stat) (top-10 (last stat))))
+ (print-report stats))))
+
+(comment
+ (declare test-data)
+
+ (deftest test-wf2 ;; it's a smoke test not a unit test
+ (let [so-far (agent {})
+ per-line (fn [accum line] (proc-line line so-far accum))
+ accum-1 (reduce per-line nil test-data)
+ accum-2 (proc-line nil so-far accum-1)
+ stats (report-wf2 so-far)
+ ]
+ (do
+ (is (= (stats :404s) '(["/ongoing/serif.css" 2]
["/ongoing/When/200x/2004/04/27/-/W3C/DTD%20XHTML%201.1/EN" 1])))
+ (is (= (stats :bytes) '(["/ongoing/ongoing.atom" 627776]
["/ongoing/door60.jpg" 548800] ["/ongoing/When/200x/2003/05/06/Rune.png"
293768] ["/ongoing/When/200x/2003/05/06/Glacier.png" 228856]
["/ongoing/When/200x/2003/12/07/JVaughan.png" 189428]
["/ongoing/When/200x/2007/03/29/IMG_0059.png" 169356]
["/ongoing/When/200x/2005/08/05/IMG_3937.png" 152739]
["/ongoing/When/200x/2007/06/15/IMGP5832.png" 131839] ["/ongoing/potd.png"
80265] ["/ongoing/sans.css" 28932])))
+ (is (= (stats :hits) '(["/ongoing/When/200x/2007/06/17/Fathers-Day"
3] ["/ongoing/When/200x/2003/10/16/Debbie" 1]
["/ongoing/When/200x/2003/09/03/McLaren" 1]
["/ongoing/When/200x/2007/03/29/Maxed" 1]
["/ongoing/When/200x/2004/04/27/RSSticker" 1]
["/ongoing/When/200x/2003/05/06/2000Pix" 1]
["/ongoing/When/200x/2006/08/25/Startup-Camp" 1]
["/ongoing/When/200x/2007/06/15/Cameras" 1])))
+ (is (= (stats :fetchers) '(["proxy.ricoh.co.jp" 2]
["cpc2-swan1-0-0-cust781.swan.cable.ntl.com" 1]
["201-34-233-57.bsace704.dsl.brasiltelecom.net.br" 1]
["pool-71-175-67-187.phlapa.fios.verizon.net" 1]
["ras75-3-82-225-164-105.fbx.proxad.net" 1]
["82-208-115-51.cs3-ats34-dzer.dialup.mts-nn.ru" 1]
["adsl-074-229-029-232.sip.mia.bellsouth.net" 1] ["host-0014.tailrank.com" 1]
["lj910366.inktomisearch.com" 1])))
+ (is (= (stats :referers)
'(["\"http://cx1.src.ricoh.co.jp/wordpress/archives/213\"" 2] ;
["\"http://search.virginmedia.com/results/index.php?channel=other&q=debbie+does&cr=\"";
1] ["\"http://www.tbray.org/ongoing/\"" 1] ["\"http://blogs.sun.com/\"" 1] ;
["\"http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&hl=en&start=31&um=1&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN\"";
1])))
+ )))
+
+ (def test-data
+ ["gmp-ea-fw-1.sun.com - - [17/Jun/2007:22:45:55 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"Mozilla/5.0 (X11; U; SunOS
sun4v; en-US; rv:1.8.0.5) Gecko/20060731 Firefox/1.5.0.5\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:45:56 -0700] \"GET
/ongoing/When/200x/2007/06/15/Cameras HTTP/1.0\" 200 8152
\"http://cx1.src.ricoh.co.jp/wordpress/archives/213\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "lj910366.inktomisearch.com - - [17/Jun/2007:22:45:57 -0700] \"GET
/ongoing/When/200x/2006/08/25/Startup-Camp HTTP/1.0\" 200 5021 \"-\"
\"Mozilla/5.0 (compatible; Yahoo! Slurp China;
http://misc.yahoo.com.cn/help.html)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:45:57 -0700] \"GET
/ongoing/ongoing.js HTTP/1.0\" 200 6688
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:45:57 -0700] \"GET
/ongoing/sans.css HTTP/1.0\" 200 4822
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "host-0014.tailrank.com - - [17/Jun/2007:22:45:58 -0700] \"GET
/ongoing/When/200x/2007/06/17/Fathers-Day HTTP/1.1\" 200 8251 \"-\"
\"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2.1; aggregator:Tailrank;
http://tailrank.com/robot) Gecko/20021130\""
+ "relay1.ops.re4.yahoo.net - - [17/Jun/2007:22:45:58 -0700] \"GET
/ongoing/ongoing.rss HTTP/1.0\" 301 315 \"-\" \"Yahoo Pipes 1.0\""
+ "relay1.ops.re4.yahoo.net - - [17/Jun/2007:22:45:58 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.0\" 304 - \"-\" \"Yahoo Pipes 1.0\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:45:58 -0700] \"GET
/ongoing/serif.css HTTP/1.0\" 200 4808
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "207.241.145.68 - - [17/Jun/2007:22:45:58 -0700] \"GET
/ongoing/ongoing.rss HTTP/1.1\" 301 322 \"-\" \"blogrunner-reaper
(+http://www.blogrunner.com/)\""
+ "201-34-233-57.bsace704.dsl.brasiltelecom.net.br - -
[17/Jun/2007:22:45:58 -0700] \"GET /ongoing/What/Arts/Movies/ HTTP/1.1\" 200
12847 \"-\" \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:45:59 -0700] \"GET
/ongoing/Feed.png HTTP/1.0\" 200 786
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:45:59
-0700] \"GET /ongoing/When/200x/2003/05/06/2000Pix HTTP/1.1\" 200 6302
\"http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&hl=en&start=31&um=1&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN\";
\"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12)
Gecko/20070508 Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:45:59
-0700] \"GET /ongoing/sans.css HTTP/1.1\" 200 4822
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:45:59
-0700] \"GET /ongoing/serif.css HTTP/1.1\" 200 4808
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:45:59
-0700] \"GET /ongoing/ongoing.js HTTP/1.1\" 200 6688
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:00 -0700] \"GET
/ongoing/door60.jpg HTTP/1.0\" 200 137200
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:00 -0700] \"GET
/ongoing/When/200x/2007/06/15/IMGP5832.png HTTP/1.0\" 200 131839
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:00
-0700] \"GET /ongoing/When/200x/2003/05/06/Rune.png HTTP/1.1\" 200 143944
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:00
-0700] \"GET /ongoing/When/200x/2003/05/06/Glacier.png HTTP/1.1\" 200 78117
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:00
-0700] \"GET /ongoing/When/200x/2003/05/06/-big/Glacier.jpg HTTP/1.1\" 403
325
\"http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&hl=en&um=1&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN&frame=small\";
\"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12)
Gecko/20070508 Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:00
-0700] \"GET /ongoing/Feed.png HTTP/1.1\" 200 786
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "58x80x72x227.ap58.ftth.ucom.ne.jp - - [17/Jun/2007:22:46:00 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"RssBar/1.29\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:01
-0700] \"GET /favicon.ico HTTP/1.1\" 200 318 \"-\" \"Mozilla/5.0 (Windows; U;
Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508 Firefox/1.5.0.12\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:01 -0700] \"GET
/ongoing/in-feed.xml HTTP/1.0\" 200 977
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:01 -0700] \"GET
/ongoing/picInfo.xml?o=http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras
HTTP/1.0\" 200 196
\"http://www.tbray.org/ongoing/When/200x/2007/06/15/Cameras\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "c-24-21-119-91.hsd1.mn.comcast.net - - [17/Jun/2007:22:46:01 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"Windows-RSS-Platform/1.0
(MSIE 7.0; Windows NT 6.0)\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:01
-0700] \"GET /ongoing/door60.jpg HTTP/1.1\" 200 137200
\"http://www.tbray.org/ongoing/serif.css\" \"Mozilla/5.0 ;(Windows; U; Windows
NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508 Firefox/1.5.0.12\""
+ "c-24-4-61-178.hsd1.ca.comcast.net - - [17/Jun/2007:22:46:01 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"AppleSyndication/54\""
+ "mt501023.inktomisearch.com - - [17/Jun/2007:22:46:02 -0700] \"GET
/ongoing/When/200x/2007/ HTTP/1.0\" 304 - \"-\" \"Mozilla/5.0 (compatible;
Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\""
+ "70-56-93-224.tukw.qwest.net - - [17/Jun/2007:22:46:03 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"Vienna/2.1.2.2110\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:03
-0700] \"GET /ongoing/When/200x/2003/05/06/Glacier.png HTTP/1.1\" 206 150739
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:03
-0700] \"GET /ongoing/When/200x/2003/05/06/Rune.png HTTP/1.1\" 206 149824
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:03
-0700] \"GET /ongoing/Feed.png HTTP/1.1\" 200 786
\"http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix\" \"Mozilla/5.0 ;
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:04
-0700] \"GET
/ongoing/picInfo.xml?o=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix
HTTP/1.1\" 200 196 \"-\" \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US;
rv:1.8.0.12) Gecko/20070508 Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:04
-0700] \"GET /ongoing/in-feed.xml HTTP/1.1\" 200 977 \"-\" \"Mozilla/5.0
(Windows; U; Windows NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508
Firefox/1.5.0.12\""
+ "adsl-074-229-029-232.sip.mia.bellsouth.net - - [17/Jun/2007:22:46:04
-0700] \"GET /ongoing/door60.jpg HTTP/1.1\" 200 137200
\"http://www.tbray.org/ongoing/serif.css\" \"Mozilla/5.0 ;(Windows; U; Windows
NT 5.1; en-US; rv:1.8.0.12) Gecko/20070508 Firefox/1.5.0.12\""
+ "sub19-247.member.dsl-only.net - - [17/Jun/2007:22:46:04 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\" \"Mozilla/4.0 (compatible;
MSIE 7.0; Windows NT 5.1; Tablet PC 1.7; .NET CLR 1.0.3705; .NET CLR
2.0.50727; InfoPath.2; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30; MSOffice
12)\""
+ "82-208-115-51.cs3-ats34-dzer.dialup.mts-nn.ru - -
[17/Jun/2007:22:46:05 -0700] \"GET /ongoing/When/200x/2007/06/17/Fathers-Day
HTTP/1.1\" 200 8251 \"http://blogs.sun.com/\" \"Mozilla/5.0 ;(Windows; U; Win
9x 4.90; ru; rv:1.8.1.4) Gecko/20070515 Firefox/2.0.0.4 Mnenhy/0.7.5.666\""
+ "ras75-3-82-225-164-105.fbx.proxad.net - - [17/Jun/2007:22:46:06 -0700]
\"GET /ongoing/When/200x/2007/06/17/Fathers-Day HTTP/1.1\" 200 8251
\"http://www.tbray.org/ongoing/\" \"Mozilla/5.0 ;(Macintosh; U; Intel Mac OS
X; en-US; rv:1.8.1.4) Gecko/20070515 Firefox/2.0.0.4\""
+ "mt501023.inktomisearch.com - - [17/Jun/2007:22:46:06 -0700] \"GET
/ongoing/sans.css HTTP/1.0\" 200 4822
\"http://www.tbray.org/ongoing/When/200x/2007/\" \"Mozilla/5.0 ;(compatible;
Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\""
+ "ras75-3-82-225-164-105.fbx.proxad.net - - [17/Jun/2007:22:46:07 -0700]
\"GET /ongoing/sans.css HTTP/1.1\" 304 -
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.4)
Gecko/20070515 Firefox/2.0.0.4\""
+ "ras75-3-82-225-164-105.fbx.proxad.net - - [17/Jun/2007:22:46:07 -0700]
\"GET /ongoing/serif.css HTTP/1.1\" 304 -
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.4)
Gecko/20070515 Firefox/2.0.0.4\""
+ "ras75-3-82-225-164-105.fbx.proxad.net - - [17/Jun/2007:22:46:07 -0700]
\"GET /ongoing/ongoing.js HTTP/1.1\" 304 -
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.4)
Gecko/20070515 Firefox/2.0.0.4\""
+ "ras75-3-82-225-164-105.fbx.proxad.net - - [17/Jun/2007:22:46:07 -0700]
\"GET /ongoing/potd.png HTTP/1.1\" 304 -
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.4)
Gecko/20070515 Firefox/2.0.0.4\""
+ "82-208-115-51.cs3-ats34-dzer.dialup.mts-nn.ru - -
[17/Jun/2007:22:46:07 -0700] \"GET /ongoing/sans.css HTTP/1.1\" 200 4822
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Windows; U; Win 9x 4.90; ru; rv:1.8.1.4) Gecko/20070515
Firefox/2.0.0.4 Mnenhy/0.7.5.666\""
+ "ras75-3-82-225-164-105.fbx.proxad.net - - [17/Jun/2007:22:46:07 -0700]
\"GET
/ongoing/picInfo.xml?o=http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day
HTTP/1.1\" 200 196
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US; rv:1.8.1.4)
Gecko/20070515 Firefox/2.0.0.4\""
+ "techmeme.com - - [17/Jun/2007:22:46:11 -0700] \"GET /robots.txt
HTTP/1.1\" 200 1364 \"-\" \"Mozilla/5.0 (compatible; Wazzup1.0.6706;
http://70.86.131.10/Wazzup)\""
+ "techmeme.com - - [17/Jun/2007:22:46:11 -0700] \"GET /robots.txt
HTTP/1.1\" 200 1364 \"-\" \"Mozilla/5.0 (compatible; Wazzup1.0.3199;
http://70.86.131.10/Wazzup)\""
+ "crm11.image.search.mud.yahoo.net - - [17/Jun/2007:22:46:12 -0700]
\"GET /ongoing/When/200x/2005/08/05/IMG_3937.png HTTP/1.0\" 200 152739 \"-\"
\"Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot
com)\""
+ "techmeme.com - - [17/Jun/2007:22:46:12 -0700] \"GET /ongoing/
HTTP/1.1\" 200 19032 \"-\" \"Mozilla/5.0 (compatible; Wazzup1.0.7330;
http://70.86.131.10/Wazzup)\""
+ "techmeme.com - - [17/Jun/2007:22:46:12 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\" \"Mozilla/5.0 (compatible;
Wazzup1.0.1212; http://70.86.131.10/Wazzup)\""
+ "dsl081-128-098.nyc1.dsl.speakeasy.net - - [17/Jun/2007:22:46:12 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\" \"Mozilla/4.0
(compatible; MSIE 7.0; Windows NT 5.1; SU 3.002; MSDigitalLocker; .NET CLR
1.1.4322; InfoPath.2; .NET CLR 2.0.50727; MSOffice 12)\""
+ "pool-71-175-67-187.phlapa.fios.verizon.net - - [17/Jun/2007:22:46:14
-0700] \"GET /ongoing/When/200x/2004/04/27/RSSticker HTTP/1.1\" 200 5310
\"-\" \"-\""
+ "pool-71-175-67-187.phlapa.fios.verizon.net - - [17/Jun/2007:22:46:14
-0700] \"GET /ongoing/When/200x/2004/04/27/-/W3C/DTD%20XHTML%201.1/EN
HTTP/1.1\" 404 327 \"-\" \"-\""
+ "mt501023.inktomisearch.com - - [17/Jun/2007:22:46:15 -0700] \"GET
/ongoing/serif.css HTTP/1.0\" 404 4808
\"http://www.tbray.org/ongoing/When/200x/2007/\" \"Mozilla/5.0 ;(compatible;
Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)\""
+ "host100.newsgator.com - - [17/Jun/2007:22:46:16 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"NewsGatorOnline/2.0
(http://www.newsgator.com; 7088 subscribers)\""
+ "80.86.78.228 - - [17/Jun/2007:22:46:16 -0700] \"GET /ongoing/
HTTP/1.1\" 304 - \"http://feeds.feedburner.com/imurdock\" \"Sunrise/0.42j ;
(Linux)\""
+ "bzq-88-153-172-148.red.bezeqint.net - - [17/Jun/2007:22:46:19 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"NetNewsWire/2.1b37 (Mac
OS X; Lite; http://ranchero.com/netnewswire/)\""
+ "208.68.164.11 - - [17/Jun/2007:22:46:20 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\" \"XML-FeedPP/0.21
XML-TreePP/0.19 libwww-perl/5.805\""
+ "newest.xs4all.nl - - [17/Jun/2007:22:46:20 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"NetNewsWire/2.1.1 (Mac OS X;
Lite; http://ranchero.com/netnewswire/)\""
+ "c-24-21-119-91.hsd1.mn.comcast.net - - [17/Jun/2007:22:46:24 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"Mozilla/4.0 (compatible;
MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.0.04506;
InfoPath.2; MSOffice 12)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:24 -0700] \"GET
/ongoing/When/200x/2007/03/29/Maxed HTTP/1.0\" 200 9583
\"http://cx1.src.ricoh.co.jp/wordpress/archives/213\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "chello084113131197.4.13.vie.surfer.at - - [17/Jun/2007:22:46:27 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"BlogBridge 5.7
(http://www.blogbridge.com/) 1.6.0_01\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:27 -0700] \"GET
/ongoing/When/200x/2007/03/29/IMG_0059.png HTTP/1.0\" 200 169356
\"http://www.tbray.org/ongoing/When/200x/2007/03/29/Maxed\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "c-76-97-239-9.hsd1.ga.comcast.net - - [17/Jun/2007:22:46:27 -0700]
\"GET /ongoing/ongoing.rss HTTP/1.1\" 301 327 \"-\" \"Mozilla/5.0 (Macintosh;
U; Intel Mac OS X; en-US; rv:1.8.1.4) Gecko/20070515 Firefox/2.0.0.4\""
+ "c-76-97-239-9.hsd1.ga.comcast.net - - [17/Jun/2007:22:46:27 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"Mozilla/5.0 (Macintosh;
U; Intel Mac OS X; en-US; rv:1.8.1.4) Gecko/20070515 Firefox/2.0.0.4\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:28 -0700] \"GET
/ongoing/picInfo.xml?o=http://www.tbray.org/ongoing/When/200x/2007/03/29/Maxed
HTTP/1.0\" 200 196
\"http://www.tbray.org/ongoing/When/200x/2007/03/29/Maxed\" \"Mozilla/4.0 ;
(compatible; MSIE 6.0; Windows NT 5.1; SV1)\""
+ "c-67-181-218-96.hsd1.ca.comcast.net - - [17/Jun/2007:22:46:29 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 301 323 \"-\" \"Mozilla/5.0 (X11; U;
Linux i686; en-US; rv:1.8.1.4) Gecko/20061201 Firefox/2.0.0.4
(Ubuntu-feisty)\""
+ "83-131-48-167.adsl.net.t-com.hr - - [17/Jun/2007:22:46:29 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"AppleSyndication/54\""
+ "24-216-68-77.dhcp.stls.mo.charter.com - - [17/Jun/2007:22:46:30 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"NetNewsWire/2.1.1 (Mac
OS X; http://ranchero.com/netnewswire/)\""
+ "c-67-181-218-96.hsd1.ca.comcast.net - - [17/Jun/2007:22:46:30 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\" \"Mozilla/5.0 (X11; U;
Linux i686; en-US; rv:1.8.1.4) Gecko/20061201 Firefox/2.0.0.4
(Ubuntu-feisty)\""
+ "82-208-115-51.cs3-ats34-dzer.dialup.mts-nn.ru - -
[17/Jun/2007:22:46:31 -0700] \"GET /ongoing/serif.css HTTP/1.1\" 404 4808
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Windows; U; Win 9x 4.90; ru; rv:1.8.1.4) Gecko/20070515
Firefox/2.0.0.4 Mnenhy/0.7.5.666\""
+ "crm11.image.search.mud.yahoo.net - - [17/Jun/2007:22:46:31 -0700]
\"GET /ongoing/When/200x/2003/12/07/JVaughan.png HTTP/1.0\" 200 189428 \"-\"
\"Yahoo-MMCrawler/3.x (mms dash mmcrawler dash support at yahoo dash inc dot
com)\""
+ "cpe0013464a4357-cm001225449bf4.cpe.net.cable.rogers.com - -
[17/Jun/2007:22:46:31 -0700] \"GET /ongoing/ongoing.atom HTTP/1.1\" 304 -
\"-\" \"NetNewsWire/2.1.1 (Mac OS X; http://ranchero.com/netnewswire/)\""
+ "m11b.feeds.yandex.net - - [17/Jun/2007:22:46:34 -0700] \"GET
/robots.txt HTTP/1.1\" 200 1364 \"-\" \"YandexSomething/1.0\""
+ "m11b.feeds.yandex.net - - [17/Jun/2007:22:46:34 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"YandexBlog/0.99.101
(compatible; DOS3.30; Mozilla/5.0; B; robot) 13 readers\""
+ "c-71-198-185-175.hsd1.ca.comcast.net - - [17/Jun/2007:22:46:35 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"NetNewsWire/2.1 (Mac OS
X; Lite; http://ranchero.com/netnewswire/)\""
+ "nulgw14.unisys.co.jp - - [17/Jun/2007:22:46:35 -0700] \"GET
/ongoing/sans.css HTTP/1.1\" 200 4822 \"-\" \"Mozilla/4.0 (compatible;)\""
+ "cpe001310d25021-cm001225708864.cpe.net.cable.rogers.com - -
[17/Jun/2007:22:46:36 -0700] \"GET /ongoing/ongoing.atom HTTP/1.1\" 304 -
\"-\" \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR
2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.2; Tablet PC
2.0; .NET CLR 3.5.20404; MSOffice 12)\""
+ "c-24-5-76-35.hsd1.ca.comcast.net - - [17/Jun/2007:22:46:36 -0700]
\"GET /ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\" \"NetNewsWire/2.1.1
(Mac OS X; http://ranchero.com/netnewswire/)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:38 -0700] \"GET
/ongoing/ongoing.rss HTTP/1.0\" 301 315 \"-\" \"FreshReader/2.0.07041100 (id:
549f074b; lang: ja; http://www.freshreader.com/; 1 subscribers)\""
+ "proxy.ricoh.co.jp - - [17/Jun/2007:22:46:39 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.0\" 200 69717 \"-\" \"FreshReader/2.0.07041100
(id: 549f074b; lang: ja; http://www.freshreader.com/; 1 subscribers)\""
+ "82-208-115-51.cs3-ats34-dzer.dialup.mts-nn.ru - -
[17/Jun/2007:22:46:39 -0700] \"GET /ongoing/Feed.png HTTP/1.1\" 200 786
\"http://www.tbray.org/ongoing/When/200x/2007/06/17/Fathers-Day\" ;
\"Mozilla/5.0 (Windows; U; Win 9x 4.90; ru; rv:1.8.1.4) Gecko/20070515
Firefox/2.0.0.4 Mnenhy/0.7.5.666\""
+"201-34-233-57.bsace704.dsl.brasiltelecom.net.br - - [17/Jun/2007:22:46:41
-0700] \"GET /ongoing/When/200x/2003/09/03/McLaren HTTP/1.1\" 200 6610 \"-\"
\"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)\""
+"1a.c3.5d45.static.theplanet.com - - [17/Jun/2007:22:46:43 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.0\" 304 - \"-\" \"Gregarius/0.5.4
(+http://devlog.gregarius.net/docs/ua)\""
+"p57b646bd.dip0.t-ipconnect.de - - [17/Jun/2007:22:46:44 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.0\" 304 - \"-\" \"NetNewsWire/2.0.1 (Mac OS X;
http://ranchero.com/netnewswire/)\""
+"p3ee3d6f8.dip.t-dialin.net - - [17/Jun/2007:22:46:46 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 304 - \"-\" \"NetNewsWire/3.0 (Mac OS X;
http://ranchero.com/netnewswire/)\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:48 -0700]
\"GET /ongoing/When/200x/2003/10/16/Debbie HTTP/1.1\" 200 8244
\"http://search.virginmedia.com/results/index.php?channel=other&q=debbie+does&cr=\";
\"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0
(compatible; MSIE 6.0; Windows NT 5.1; SV1) )\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:48 -0700]
\"GET /ongoing/sans.css HTTP/1.1\" 200 4822
\"http://www.tbray.org/ongoing/When/200x/2003/10/16/Debbie\" \"Mozilla/4.0 ;
(compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1) )\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:48 -0700]
\"GET /ongoing/serif.css HTTP/1.1\" 200 4808
\"http://www.tbray.org/ongoing/When/200x/2003/10/16/Debbie\" \"Mozilla/4.0 ;
(compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1) )\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:48 -0700]
\"GET /ongoing/ongoing.js HTTP/1.1\" 200 6688
\"http://www.tbray.org/ongoing/When/200x/2003/10/16/Debbie\" \"Mozilla/4.0 ;
(compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1) )\""
+"slim07.kataweb.it - - [17/Jun/2007:22:46:48 -0700] \"GET
/ongoing/ongoing.rss HTTP/1.1\" 301 322 \"-\" \"Java/1.4.2_08\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:48 -0700]
\"GET /ongoing/Feed.png HTTP/1.1\" 200 786
\"http://www.tbray.org/ongoing/When/200x/2003/10/16/Debbie\" \"Mozilla/4.0 ;
(compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1) )\""
+"slim07.kataweb.it - - [17/Jun/2007:22:46:48 -0700] \"GET
/ongoing/ongoing.rss HTTP/1.1\" 301 327 \"-\" \"Java/1.4.2_08\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:49 -0700]
\"GET /ongoing/potd.png HTTP/1.1\" 200 80265
\"http://www.tbray.org/ongoing/When/200x/2003/10/16/Debbie\" \"Mozilla/4.0 ;
(compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1) )\""
+"host86-143-230-228.range86-143.btcentralplus.com - - [17/Jun/2007:22:46:49
-0700] \"GET /ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\"
\"NetNewsWire/2.1.1 (Mac OS X; Lite; http://ranchero.com/netnewswire/)\""
+"slim07.kataweb.it - - [17/Jun/2007:22:46:49 -0700] \"GET
/ongoing/ongoing.atom HTTP/1.1\" 200 69717 \"-\" \"Java/1.4.2_08\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:49 -0700]
\"GET /ongoing/door60.jpg HTTP/1.1\" 200 137200
\"http://www.tbray.org/ongoing/When/200x/2003/10/16/Debbie\" \"Mozilla/4.0 ;
(compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0;
Windows NT 5.1; SV1) )\""
+"cpc2-swan1-0-0-cust781.swan.cable.ntl.com - - [17/Jun/2007:22:46:50 -0700]
\"GET /favicon.ico HTTP/1.1\" 200 318 \"-\" \"Mozilla/4.0 (compatible; MSIE
7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)
)\""
+])
+)
+
Index: src/org/tbray/JOverhead.java
===================================================================
--- src/org/tbray/JOverhead.java (revision 3)
+++ src/org/tbray/JOverhead.java (revision 4)
@@ -5,6 +5,7 @@
package org.tbray;
+import java.io.File;
import java.io.FileInputStream;
import java.nio.CharBuffer;
import java.nio.MappedByteBuffer;
@@ -18,12 +19,22 @@
*/
public class JOverhead {
- private static void blok(FileChannel channel, long offset, int size)
throws Exception {
- MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY,
offset, size);
- CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder();
- CharBuffer chars = decoder.decode(buffer);
- char c = chars.get(chars.length() - 10);
- System.out.println("C=" + c);
+ static Charset charset = Charset.forName("UTF-8");
+
+ private static int[] blok(FileInputStream in, int size) throws Exception
{
+ byte[] buffer = new byte[size];
+ int read = in.read(buffer);
+ String text = new String(buffer, 0, read, charset);
+ int lines = 0;
+ int chars = 0;
+ for (int last = 0, next = text.indexOf('\n', last); next >= 0; next
= text.indexOf('\n', last)) {
+ String line = text.substring(last, next);
+ lines++;
+ chars += line.length();
+ last = next + 1;
+ }
+ int[] ret = { lines, chars };
+ return ret;
}
/**
@@ -31,19 +42,27 @@
*/
public static void main(String[] args) throws Exception {
// arg is filename
- FileChannel channel = (new FileInputStream(args[0])).getChannel();
- long dataSize = channel.size();
+ File file = new File(args[0]);
+ FileInputStream in = new FileInputStream(file);
+ long dataSize = file.length();
int blockSize = 10 * 1024 * 1024;
if (args.length > 1)
blockSize = Integer.parseInt(args[1]) * 1024 * 1024;
long blockCount = dataSize / blockSize;
int tailSize = (int) (dataSize % blockSize);
+ long lines = 0, chars = 0;
+ int[] r;
for (long i = 0; i < blockCount; i++) {
System.out.println("B " + i);
- blok(channel, i * blockSize, blockSize);
+ r = blok(in, blockSize);
+ lines += r[0];
+ chars += r[1];
}
- blok(channel, blockCount * blockSize, tailSize);
+ r = blok(in, tailSize);
+ lines += r[0];
+ chars += r[1];
+ System.out.println("L=" + lines + " C=" + chars);
}
}
Index: src/org/tbray/popular.clj
===================================================================
--- src/org/tbray/popular.clj (revision 3)
+++ src/org/tbray/popular.clj (revision 4)
@@ -1,10 +1,5 @@
+(ns org.tbray.popular)
-(ns org.tbray.paralines
- ;(:require )
- ;(:use )
- ;(:import )
- )
-
(def re #"GET /ongoing/When/\d\d\dx/(\d\d\d\d/\d\d/\d\d/[^ .]+) ")
(defn record [target accum]
@@ -14,7 +9,7 @@
(assoc accum target 1))
{ target 1 }))
-(defn proc-line [ line so-far accum]
+(defn proc-line [line so-far accum]
(if line
(if-let [[_ target] (re-find re line)]
(record target accum)
Index: src/org/tbray/read_lines.clj
===================================================================
--- src/org/tbray/read_lines.clj (revision 0)
+++ src/org/tbray/read_lines.clj (revision 4)
@@ -0,0 +1,104 @@
+(ns org.tbray.read-lines
+ (:import (java.io FileInputStream) (java.nio.charset Charset)))
+
+(declare dispatch proc-block)
+(def *charset* (. java.nio.charset.Charset forName "UTF-8"))
+
+(defn frag
+ "fragments is a map whose keys are of the form (str index :front|:back)"
+ [fragments fragment index front-back]
+ (dosync
+ (ref-set fragments (assoc @fragments (str index front-back) (new String
fragment)))))
+
+(defn join-fragments [fragments-in params]
+ (loop [fragments fragments-in accum nil]
+ (if (not (empty? fragments))
+ (let [[key frag] (first fragments)
+ [istring fb] (. #":" split key)
+ index (new Integer istring)
+ is-front (= fb "front")
+ other-key (if is-front (str (dec index) ":back") (str (inc
index) ":front"))
+ other (fragments other-key)
+ line (if is-front (str other frag) (str frag other))
+ next-accum ((params :per-line) accum line) ]
+ (recur (dissoc fragments key other-key) next-accum))
+ ((params :done-lines) accum))))
+
+(defn count-down [so-far params]
+ (let [processed (inc so-far)]
+ (if (> processed (params :block-count))
+ (let [fragments (params :fragments)] ; yep, all finished
+ (join-fragments (deref fragments) params)
+ ((params :all-done))
+ (. System exit 0))
+ processed)))
+
+(defn proc-block [bytes index params last]
+ (println (str "Block " index))
+ (let [text (new String bytes *charset*)
+ trailing-nl (. text endsWith "\n")
+ chunks (. #"\n" split text)
+ front (first chunks)
+ lc (get chunks (dec (count chunks))) ; (last chunks) tickles a
Clojure 1.0 bug
+ back (if trailing-nl "" lc)
+ lines (if trailing-nl (rest chunks) (butlast (rest chunks)))
+ accumulator (reduce (params :per-line) nil lines) ] ; all the work
happens here
+
+ ((params :done-lines) accumulator) ; tell user-provided function this
thread done
+ (frag (params :fragments) front index :front) ; save front/back fragments
+ (if (not last)
+ (frag (params :fragments) back index :back))
+
+ (send (params :processed) count-down params) ; record one more block
done
+ (send (params :index-agent) dispatch params))) ; tell dispatcher we're
done
+
+(defn dispatch [ index params ] ; reading is single-threaded here
+ (let [next (inc index) block-count (params :block-count)]
+ (if (<= index block-count)
+ (let [last (= index block-count)
+ size (if last (params :tail-size) (params :block-size))
+ buffer (make-array Byte/TYPE size)]
+ (. (params :input) read buffer)
+ (.start (new Thread #(proc-block buffer index params last)))
+ next)
+ index)))
+
+(defn read-lines
+ "Reads the named file and feeds the lines one at a time to the function
+'destination', which will be called in parallel on many threads with three
+arguments, the first being a line from the file, the second 'user-data',
+the third an accumulator, whose value will be nil on the first call in
+any given thread. The destination function's return value will appear on the
+next call as its accumulator argument. If the function is called with a nil
+value for the first (line) argument, that signals the final call in some
given
+thread, so that the function may choose to do wind-up and reporting work.
+
+When all the lines have been processed, the the 'all-done' function will be
+called with 'user-data' as its single argument. The 'destination' function
+will be called on multiple threads in parallel, so its logic must be
+thread-safe. There is no guarantee in which order the lines from the file
+will be processed. The 'settings' argument is a map; the default values for
the
+number of threads and file block-size may be overridden with the :width and
+:block-size values respectively."
+ [file-name destination all-done user-data settings]
+ (let [block-size (or (settings :block-size) (* 10 1024 1024))
+ input (new FileInputStream file-name)
+ data-size (. (new java.io.File file-name) length)
+ fragments (ref {})
+ block-count (quot data-size block-size)
+ thread-count (max 1 (min block-count (or (settings :width) 10)))
+ tail-size (mod data-size block-size)
+ index (agent 0)
+ params {:block-count block-count :block-size block-size :tail-size
tail-size
+ :input input :processed (agent 0)
+ :per-line (fn [accum #^String line] (destination (new String
line) user-data accum))
+ :done-lines (fn [accum] (destination nil user-data accum))
+ :all-done (fn [] (all-done user-data))
+ :fragments fragments
+ :index-agent index }]
+
+ (frag fragments "" -1 :back) ; prime the fragments pump
+ (doseq [_ (repeat thread-count 0)]
+ (send index dispatch params))
+ (. (. Thread currentThread) join)))
+
Property changes on: .
___________________________________________________________________
Added: svn:ignore
+ build
|
[divide-and-conquer~concur-next:4] State of play as of Dec. 9th. |
TimBray | 12/10/2009 |
![http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&hl=en&start=31&um=1&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN\""](http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&hl=en&start=31&um=1&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN\""){kind=link}
![http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&hl=en&start=31&um=1&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN\"](http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&hl=en&start=31&um=1&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN\"){kind=link}
![http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&hl=en&um=1&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN&frame=small\"](http://images.google.com/imgres?imgurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/-big/Glacier.jpg&imgrefurl=http://www.tbray.org/ongoing/When/200x/2003/05/06/2000Pix&h=642&w=960&sz=298&tbnid=vrql887CTfKbeM:&tbnh=99&tbnw=148&hl=en&um=1&prev=/images%3Fq%3Dglacier%2Bbay%26start%3D20%26ndsp%3D20%26svnum%3D10%26um%3D1%26hl%3Den%26client%3Dfirefox-a%26rls%3Dorg.mozilla:en-US:official%26sa%3DN&frame=small\"){kind=link}