Bug #667013 “improve report-status.sh” : Bugs : Archive Widecrawl

Revision history for this message

siznax (siznax) wrote on 2010-10-26:

#1

Download full text (5.4 KiB)

On 10/25/10 2:04 PM, Kenji Nagahashi wrote:
> Hi Steve,
>
> Reading report-status*.sh, I came up with some ideas to simplify the code.
> As it is easier to present actual code, I attached a patch for those files.
>
> Changes also include adding "--max-time 30" option to curl invocation,
> as it seems reading job status page sooo long when it is actively crawling.
>
> Hope you find this useful.
> --Kenji

diff --git a/report-status-cluster.sh b/report-status-cluster.sh
index 7b7e339..d7a8e01 100755
--- a/report-status-cluster.sh
+++ b/report-status-cluster.sh
@@ -2,18 +2,14 @@
# siznax 2010

cluster=/home/steve/crawling/live/cluster.txt
-status=/home/steve/crawling/live/report-status.sh
-nodes=`cut -d ' ' -f 1 $cluster`
+status=`dirname $0`/report-status.sh

-self=`echo $0 | tr '/' ' ' | awk '{print $NF}'`
+self=`basename $0`

echo $self `date`

-for node in $nodes
-do
- host=`grep $node $cluster | awk '{print $1}'`
- port=`grep $node $cluster | awk '{print $2}'`
- job=` grep $node $cluster | awk '{print $3}'`
- auth=`grep $node $cluster | awk '{print $4}'`
- $status $host $port $job $auth 0
-done
+exec 3<$cluster
+while read -u 3 host port job auth; do
+ $status $host $port $job $auth 0
+done
+exec 3<&-
diff --git a/report-status.sh b/report-status.sh
index ec978e8..0470905 100755
--- a/report-status.sh
+++ b/report-status.sh
@@ -17,6 +17,8 @@ version="r0"
LC_ALL=en_US.UTF-8 # for thousands separator
hs_errs=/home/steve/crawling/live/hs_errs.txt

+verbose=false
+
if [ $# -lt 5 ]
then
     echo "Usage: $script host port job auth verbose"
@@ -26,19 +28,19 @@ else
     port=$2
     job=$3
     auth=$4
- verbose=$5
+ [ $5 == 0 ] || verbose=true
fi

-tags=('<statusDescription>'\
- '<totalUriCount>'\
- '<downloadedUriCount>'\
- '<novel>'\
- '<currentDocsPerSecond>'\
- '<averageDocsPerSecond>'\
- '<currentKiBPerSec>'\
- '<averageKiBPerSec>'\
- '<elapsedPretty>'\
- '<launchCount>')
+tags=('statusDescription'\
+ 'totalUriCount'\
+ 'downloadedUriCount'\
+ 'novel'\
+ 'currentDocsPerSecond'\
+ 'averageDocsPerSecond'\
+ 'currentKiBPerSec'\
+ 'averageKiBPerSec'\
+ 'elapsedPretty'\
+ 'launchCount')
keys=(status\
total\
downloaded\
@@ -53,18 +55,20 @@ keys=(status\
#---------------------------------------------------------------

function get_tag_value {
- cmd="grep '$tag' $tmpfile\
- | tr '<' ' '\
- | tr '>' ' '\
- | tr -s ' '\
- | cut -d ' ' -f 3"
- value=`eval $cmd`
+ value=`sed -ne "\%.*<$tag>$.*$</$tag>.*%{s//\1/p;q}" $tmpfile`
+# cmd="grep '$tag' $tmpfile | tr -s '<>' ' '
+# | tr '<' ' '\
+# | tr '>' ' '\
+# | tr -s ' '\
+# | cut -d ' ' -f 3"
+# value=`eval $cmd`
}
function get_status {
- value=`grep statusDescription $tmpfile\
- | grep -o '>[^\<]*'\
- | awk '{print $NF}'\
- | head -1`
+ value=`sed -ne '/.*<statusDescription>.*: $[^<]*$.*/{s//\1/p;q}' $tmpfile`
+# value=`grep statusDescription $tmpfile\
+# | grep -o '>[^\<]*'\
+# | awk '{print $NF}'\
+# ...

On 10/25/10 2:04 PM, Kenji Nagahashi wrote:
> Hi Steve,
> 
> Reading report-status*.sh, I came up with some ideas to simplify the code.
> As it is easier to present actual code, I attached a patch for those files.
> 
> Changes also include adding "--max-time 30" option to curl invocation, 
> as it seems reading job status page sooo long when it is actively crawling.
> 
> Hope you find this useful.
> --Kenji

diff --git a/report-status-cluster.sh b/report-status-cluster.sh
index 7b7e339..d7a8e01 100755
--- a/report-status-cluster.sh
+++ b/report-status-cluster.sh
@@ -2,18 +2,14 @@
 # siznax 2010
 
 cluster=/home/steve/crawling/live/cluster.txt
-status=/home/steve/crawling/live/report-status.sh
-nodes=`cut -d ' ' -f 1 $cluster`
+status=`dirname $0`/report-status.sh
 
-self=`echo $0 | tr '/' ' ' | awk '{print $NF}'`
+self=`basename $0`
 
 echo $self `date`
 
-for node in $nodes
-do
-    host=`grep $node $cluster | awk '{print $1}'`
-    port=`grep $node $cluster | awk '{print $2}'`
-    job=` grep $node $cluster | awk '{print $3}'`
-    auth=`grep $node $cluster | awk '{print $4}'`
-    $status $host $port $job $auth 0
-done 
+exec 3<$cluster
+while read -u 3 host port job auth; do
+  $status $host $port $job $auth 0
+done
+exec 3<&-
diff --git a/report-status.sh b/report-status.sh
index ec978e8..0470905 100755
--- a/report-status.sh
+++ b/report-status.sh
@@ -17,6 +17,8 @@ version="r0"
 LC_ALL=en_US.UTF-8  # for thousands separator
 hs_errs=/home/steve/crawling/live/hs_errs.txt
 
+verbose=false
+
 if [ $# -lt 5 ]
 then
     echo "Usage: $script host port job auth verbose"
@@ -26,19 +28,19 @@ else
     port=$2
     job=$3
     auth=$4
-    verbose=$5
+    [ $5 == 0 ] || verbose=true
 fi
 
-tags=('<statusDescription>'\
-      '<totalUriCount>'\
-      '<downloadedUriCount>'\
-      '<novel>'\
-      '<currentDocsPerSecond>'\
-      '<averageDocsPerSecond>'\
-      '<currentKiBPerSec>'\
-      '<averageKiBPerSec>'\
-      '<elapsedPretty>'\
-      '<launchCount>')
+tags=('statusDescription'\
+      'totalUriCount'\
+      'downloadedUriCount'\
+      'novel'\
+      'currentDocsPerSecond'\
+      'averageDocsPerSecond'\
+      'currentKiBPerSec'\
+      'averageKiBPerSec'\
+      'elapsedPretty'\
+      'launchCount')
 keys=(status\
       total\
       downloaded\
@@ -53,18 +55,20 @@ keys=(status\
 #---------------------------------------------------------------
 
 function get_tag_value {
-    cmd="grep '$tag' $tmpfile\
-         | tr '<' ' '\
-         | tr '>' ' '\
-         | tr -s  ' '\
-         | cut -d ' ' -f 3"
-    value=`eval $cmd`
+    value=`sed -ne "\%.*<$tag>$.*$</$tag>.*%{s//\1/p;q}" $tmpfile`
+#    cmd="grep '$tag' $tmpfile | tr -s '<>' '  '
+#         | tr '<' ' '\
+#         | tr '>' ' '\
+#         | tr -s  ' '\
+#         | cut -d ' ' -f 3"
+#    value=`eval $cmd`
 }
 function get_status {
-    value=`grep statusDescription $tmpfile\
-           | grep -o '>[^\<]*'\
-           | awk '{print $NF}'\
-           | head -1`
+    value=`sed -ne '/.*<statusDescription>.*: $[^<]*$.*/{s//\1/p;q}' $tmpfile`
+#    value=`grep statusDescription $tmpfile\
+#           | grep -o '>[^\<]*'\
+#           | awk '{print $NF}'\
+#           | head -1`
 }
 
 # -- COUNT CRASHES ---------------------------------------------
@@ -80,7 +84,7 @@ fi
 date_str=`date "+%Y-%m-%dT%TZ"`
 h=`echo $host | cut -d '.' -f 1`
 url="https://${host}:${port}/engine/job/${job}"
-copts="-kLs -u $auth --anyauth -H 'Accept: application/xml' --location"
+copts="-kLs -u $auth --anyauth -H 'Accept: application/xml' --location --max-time 20"
 tmpfile=/var/tmp/status.$$
 cmd="curl $copts $url -o $tmpfile"
 eval $cmd
@@ -89,7 +93,7 @@ eval $cmd
 
 if [ $? != 0 ]
 then
-    if [ $verbose -eq 1 ]
+    if $verbose
     then
         echo "==== $job $host N/A ===="
         printf "%15s: %s\n" hs_err_count "$hs_err_count"
@@ -99,7 +103,7 @@ then
     exit 1
 fi
 
-if [ $verbose -eq 1 ]
+if $verbose
 then
     echo "$script $version" `date`
     echo "==== $job $host ===="
@@ -128,17 +132,15 @@ do
     fi
 
     # format values
-    if [ "${keys[$i]}" == total ] ||\
-       [ "${keys[$i]}" == downloaded ] ||\
-       [ "${keys[$i]}" == "KB/sec (avg)" ]
-    then
+    case "${keys[$i]}" in
+    total|downloaded|KB/sec\ $avg$)
         string=`printf "%'d" $value`
-    elif [ "${keys[$i]}" == elapsed ]
-    then
+	;;
+    elapsed)
 	value=`echo $value | sed -e s/m.*s/m/`
         string=$value
-    elif [ "${keys[$i]}" == bytes ]
-    then
+	;;
+    bytes)
 	if [ ! $value ]
         then
             # probably a bug in the crawler
@@ -149,17 +151,16 @@ do
             string=$value
         fi
         key="GB"
-    elif [ "${keys[$i]}" == "URI/sec (avg)" ]
-    then
+        ;;
+    URI/sec\ $avg$)
         string=`printf "%.3f" $value`
-    else
+        ;;
+    *)
         string=$value
-    fi
+	;;
+    esac
 
-    if [ $verbose -eq 1 ]
-    then
-        printf "%15s: %s\n" "$key" "$string"
-    fi
+    $verbose && printf "%15s: %s\n" "$key" "$string"
     values[$i]=$value
     ((i++))
 
@@ -167,7 +168,7 @@ done
 
 # -- RAW XML values --------------------------------------------
 
-if [ $verbose -eq 1 ]; then echo "values = ${values[@]}"; fi
+$verbose && echo "values = ${values[@]}"
 
 # -- WIKI_TEXT -------------------------------------------------
 
@@ -189,7 +190,7 @@ K=`printf "%'d" ${values[6]}`
 k=`printf "%'d" ${values[7]}`
 n=${values[9]}
 
-if [ $verbose -eq 1 ]
+if $verbose
 then
     echo "wikitext = | | |${s}|${d}|${e}|${G}|${l}|${U} (${u})|${K} (${k})| |"
 fi

Revision history for this message

siznax (siznax) wrote on 2010-10-26:

#2

Download full text (5.0 KiB)

diff --git a/report-status-cluster.sh b/report-status-cluster.sh
index 7b7e339..d7a8e01 100755
--- a/report-status-cluster.sh
+++ b/report-status-cluster.sh
@@ -2,18 +2,14 @@
# siznax 2010

cluster=/home/steve/crawling/live/cluster.txt
-status=/home/steve/crawling/live/report-status.sh
-nodes=`cut -d ' ' -f 1 $cluster`
+status=`dirname $0`/report-status.sh

-self=`echo $0 | tr '/' ' ' | awk '{print $NF}'`
+self=`basename $0`

echo $self `date`

-for node in $nodes
-do
- host=`grep $node $cluster | awk '{print $1}'`
- port=`grep $node $cluster | awk '{print $2}'`
- job=` grep $node $cluster | awk '{print $3}'`
- auth=`grep $node $cluster | awk '{print $4}'`
- $status $host $port $job $auth 0
-done
+exec 3<$cluster
+while read -u 3 host port job auth; do
+ $status $host $port $job $auth 0
+done
+exec 3<&-
diff --git a/report-status.sh b/report-status.sh
index ec978e8..0470905 100755
--- a/report-status.sh
+++ b/report-status.sh
@@ -17,6 +17,8 @@ version="r0"
LC_ALL=en_US.UTF-8 # for thousands separator
hs_errs=/home/steve/crawling/live/hs_errs.txt

+verbose=false
+
if [ $# -lt 5 ]
then
     echo "Usage: $script host port job auth verbose"
@@ -26,19 +28,19 @@ else
     port=$2
     job=$3
     auth=$4
- verbose=$5
+ [ $5 == 0 ] || verbose=true
fi

-tags=('<statusDescription>'\
- '<totalUriCount>'\
- '<downloadedUriCount>'\
- '<novel>'\
- '<currentDocsPerSecond>'\
- '<averageDocsPerSecond>'\
- '<currentKiBPerSec>'\
- '<averageKiBPerSec>'\
- '<elapsedPretty>'\
- '<launchCount>')
+tags=('statusDescription'\
+ 'totalUriCount'\
+ 'downloadedUriCount'\
+ 'novel'\
+ 'currentDocsPerSecond'\
+ 'averageDocsPerSecond'\
+ 'currentKiBPerSec'\
+ 'averageKiBPerSec'\
+ 'elapsedPretty'\
+ 'launchCount')
keys=(status\
total\
downloaded\
@@ -53,18 +55,20 @@ keys=(status\
#---------------------------------------------------------------

function get_tag_value {
- cmd="grep '$tag' $tmpfile\
- | tr '<' ' '\
- | tr '>' ' '\
- | tr -s ' '\
- | cut -d ' ' -f 3"
- value=`eval $cmd`
+ value=`sed -ne "\%.*<$tag>$.*$</$tag>.*%{s//\1/p;q}" $tmpfile`
+# cmd="grep '$tag' $tmpfile | tr -s '<>' ' '
+# | tr '<' ' '\
+# | tr '>' ' '\
+# | tr -s ' '\
+# | cut -d ' ' -f 3"
+# value=`eval $cmd`
}
function get_status {
- value=`grep statusDescription $tmpfile\
- | grep -o '>[^\<]*'\
- | awk '{print $NF}'\
- | head -1`
+ value=`sed -ne '/.*<statusDescription>.*: $[^<]*$.*/{s//\1/p;q}' $tmpfile`
+# value=`grep statusDescription $tmpfile\
+# | grep -o '>[^\<]*'\
+# | awk '{print $NF}'\
+# | head -1`
}

# -- COUNT CRASHES ---------------------------------------------
@@ -80,7 +84,7 @@ fi
date_str=`date "+%Y-%m-%dT%TZ"`
h=`echo $host | cut -d '.' -f 1`
url="https://${host}:${port}/engine/job/${job}"
-copts="-kLs -u $auth --anyauth -H 'Accept: application/xml' --location"
+copts="-kLs -u $auth --anyauth -H 'Accept: application/xml' --location --max-time 20"
tmpfile=/var/tmp/status.$$
c...

diff --git a/report-status-cluster.sh b/report-status-cluster.sh
index 7b7e339..d7a8e01 100755
--- a/report-status-cluster.sh
+++ b/report-status-cluster.sh
@@ -2,18 +2,14 @@
 # siznax 2010
 
 cluster=/home/steve/crawling/live/cluster.txt
-status=/home/steve/crawling/live/report-status.sh
-nodes=`cut -d ' ' -f 1 $cluster`
+status=`dirname $0`/report-status.sh
 
-self=`echo $0 | tr '/' ' ' | awk '{print $NF}'`
+self=`basename $0`
 
 echo $self `date`
 
-for node in $nodes
-do
-    host=`grep $node $cluster | awk '{print $1}'`
-    port=`grep $node $cluster | awk '{print $2}'`
-    job=` grep $node $cluster | awk '{print $3}'`
-    auth=`grep $node $cluster | awk '{print $4}'`
-    $status $host $port $job $auth 0
-done 
+exec 3<$cluster
+while read -u 3 host port job auth; do
+  $status $host $port $job $auth 0
+done
+exec 3<&-
diff --git a/report-status.sh b/report-status.sh
index ec978e8..0470905 100755
--- a/report-status.sh
+++ b/report-status.sh
@@ -17,6 +17,8 @@ version="r0"
 LC_ALL=en_US.UTF-8  # for thousands separator
 hs_errs=/home/steve/crawling/live/hs_errs.txt
 
+verbose=false
+
 if [ $# -lt 5 ]
 then
     echo "Usage: $script host port job auth verbose"
@@ -26,19 +28,19 @@ else
     port=$2
     job=$3
     auth=$4
-    verbose=$5
+    [ $5 == 0 ] || verbose=true
 fi
 
-tags=('<statusDescription>'\
-      '<totalUriCount>'\
-      '<downloadedUriCount>'\
-      '<novel>'\
-      '<currentDocsPerSecond>'\
-      '<averageDocsPerSecond>'\
-      '<currentKiBPerSec>'\
-      '<averageKiBPerSec>'\
-      '<elapsedPretty>'\
-      '<launchCount>')
+tags=('statusDescription'\
+      'totalUriCount'\
+      'downloadedUriCount'\
+      'novel'\
+      'currentDocsPerSecond'\
+      'averageDocsPerSecond'\
+      'currentKiBPerSec'\
+      'averageKiBPerSec'\
+      'elapsedPretty'\
+      'launchCount')
 keys=(status\
       total\
       downloaded\
@@ -53,18 +55,20 @@ keys=(status\
 #---------------------------------------------------------------
 
 function get_tag_value {
-    cmd="grep '$tag' $tmpfile\
-         | tr '<' ' '\
-         | tr '>' ' '\
-         | tr -s  ' '\
-         | cut -d ' ' -f 3"
-    value=`eval $cmd`
+    value=`sed -ne "\%.*<$tag>$.*$</$tag>.*%{s//\1/p;q}" $tmpfile`
+#    cmd="grep '$tag' $tmpfile | tr -s '<>' '  '
+#         | tr '<' ' '\
+#         | tr '>' ' '\
+#         | tr -s  ' '\
+#         | cut -d ' ' -f 3"
+#    value=`eval $cmd`
 }
 function get_status {
-    value=`grep statusDescription $tmpfile\
-           | grep -o '>[^\<]*'\
-           | awk '{print $NF}'\
-           | head -1`
+    value=`sed -ne '/.*<statusDescription>.*: $[^<]*$.*/{s//\1/p;q}' $tmpfile`
+#    value=`grep statusDescription $tmpfile\
+#           | grep -o '>[^\<]*'\
+#           | awk '{print $NF}'\
+#           | head -1`
 }
 
 # -- COUNT CRASHES ---------------------------------------------
@@ -80,7 +84,7 @@ fi
 date_str=`date "+%Y-%m-%dT%TZ"`
 h=`echo $host | cut -d '.' -f 1`
 url="https://${host}:${port}/engine/job/${job}"
-copts="-kLs -u $auth --anyauth -H 'Accept: application/xml' --location"
+copts="-kLs -u $auth --anyauth -H 'Accept: application/xml' --location --max-time 20"
 tmpfile=/var/tmp/status.$$
 cmd="curl $copts $url -o $tmpfile"
 eval $cmd
@@ -89,7 +93,7 @@ eval $cmd
 
 if [ $? != 0 ]
 then
-    if [ $verbose -eq 1 ]
+    if $verbose
     then
         echo "==== $job $host N/A ===="
         printf "%15s: %s\n" hs_err_count "$hs_err_count"
@@ -99,7 +103,7 @@ then
     exit 1
 fi
 
-if [ $verbose -eq 1 ]
+if $verbose
 then
     echo "$script $version" `date`
     echo "==== $job $host ===="
@@ -128,17 +132,15 @@ do
     fi
 
     # format values
-    if [ "${keys[$i]}" == total ] ||\
-       [ "${keys[$i]}" == downloaded ] ||\
-       [ "${keys[$i]}" == "KB/sec (avg)" ]
-    then
+    case "${keys[$i]}" in
+    total|downloaded|KB/sec\ $avg$)
         string=`printf "%'d" $value`
-    elif [ "${keys[$i]}" == elapsed ]
-    then
+	;;
+    elapsed)
 	value=`echo $value | sed -e s/m.*s/m/`
         string=$value
-    elif [ "${keys[$i]}" == bytes ]
-    then
+	;;
+    bytes)
 	if [ ! $value ]
         then
             # probably a bug in the crawler
@@ -149,17 +151,16 @@ do
             string=$value
         fi
         key="GB"
-    elif [ "${keys[$i]}" == "URI/sec (avg)" ]
-    then
+        ;;
+    URI/sec\ $avg$)
         string=`printf "%.3f" $value`
-    else
+        ;;
+    *)
         string=$value
-    fi
+	;;
+    esac
 
-    if [ $verbose -eq 1 ]
-    then
-        printf "%15s: %s\n" "$key" "$string"
-    fi
+    $verbose && printf "%15s: %s\n" "$key" "$string"
     values[$i]=$value
     ((i++))
 
@@ -167,7 +168,7 @@ done
 
 # -- RAW XML values --------------------------------------------
 
-if [ $verbose -eq 1 ]; then echo "values = ${values[@]}"; fi
+$verbose && echo "values = ${values[@]}"
 
 # -- WIKI_TEXT -------------------------------------------------
 
@@ -189,7 +190,7 @@ K=`printf "%'d" ${values[6]}`
 k=`printf "%'d" ${values[7]}`
 n=${values[9]}
 
-if [ $verbose -eq 1 ]
+if $verbose
 then
     echo "wikitext = | | |${s}|${d}|${e}|${G}|${l}|${U} (${u})|${K} (${k})| |"
 fi

Revision history for this message

siznax (siznax) wrote on 2010-10-26:

#3

hm, looks like attaching a patch just puts it in another comment. sorry.

Revision history for this message

siznax (siznax) wrote on 2010-10-26:

#4

may want to move this entirely into cacti if possible

Revision history for this message

Kenji Nagahashi (knagahashi) wrote on 2011-01-15:

#5

re-implemented report-status.sh in python: report-status.py
as of Jan 3, 2011, report-status-cluster.sh runs report-status.py instead of report-status.sh

siznax (siznax) on 2011-01-19

Changed in archivewidecrawl:
assignee:	nobody → Kenji Nagahashi (knagahashi)
importance:	Undecided → Low
status:	New → Fix Committed

Archive Widecrawl

improve report-status.sh

Bug Description

Other bug subscribers

Remote bug watches