Heritrix源码分析(二) 配置文件order.xml介绍

 本博客属原创文章,欢迎转载！转载请务必注明出处:http://guoyunsky.iteye.com/blog/613412

     本博客已迁移到本人独立博客: http://www.yun5u.com/

      order.xml是整个Heritrix的核心,里面的每个一个配置都关系到Heritrix的运行情况,没读源码之前我只能从有限的渠道去获知这些配置的运用.读完之后才知道Heritrix竟然有如此灵活的运用,如可以控制抓取速度,可以优化电脑性能,可以在某一次的抓取上继续抓取.当然整个order.xml里我也没有全部掌握,只知道大部分配置的作用,希望大家指点改正以及补充,谢谢！

 

1.       代表着该抓取JOB的元素,相当于Html的meta

 <meta>
    <name>myheritrixname>
    <description>my heritrixdescription>
    <operator>Adminoperator>
    <organization>organization>
    <audience>audience>
    <date>20090520051654date>

 meta>

 

2. 跟抓取有关的所有参数,由于内容较多,并且Heritrix也已将他们分成不同模块，所以这里我也将他们拆分来说明.

 

 

  <controller>
     <string name="settings-directory">settingsstring>
     <string name="disk-path">string>
     <string name="logs-path">logsstring>
     <string name="checkpoints-path">checkpointsstring>
     <string name="state-path">statestring>
     <string name="scratch-path">scratchstring>
     <long name="max-bytes-download">0long>
     <long name="max-document-download">0long>
     <long name="max-time-sec">0long>
     <integer name="max-toe-threads">30integer>
     <integer name="recorder-out-buffer-bytes">4096integer>
     <integer name="recorder-in-buffer-bytes">65536integer>
     <integer name="bdb-cache-percent">0integer>
 <newObject name="scope" class="org.archive.crawler.deciderules.DecidingScope">
 newObject>
 <map name="http-headers">
 map>
  <newObject name="robots-honoring-policy" class="org.archive.crawler.datamodel.RobotsHonoringPolicy">
 newObject>
  <newObject name="frontier" class="org.archive.crawler.frontier.BdbFrontier">
 map>
 <map name="pre-fetch-processors">
 map>
 <map name="fetch-processors">
 map>
 <map name="extract-processors">
 map>
 <map name="write-processors">
 map>
 <map name="post-processors">
 map>
 <map name="loggers">
 map>
 <newObject name="credential-store" class="org.archive.crawler.datamodel.CredentialStore">
  newObject>
  controller>

3.接下来拆分每个组件的配置文件一一进行说明，最后对Heritrix主要的配置也就是我们可以影响抓取的配置进行说明。

   3.1：抓取范围

  

 <newObject name="scope" class="org.archive.crawler.deciderules.DecidingScope">
       <boolean name="enabled">falseboolean>
       <string name="seedsfile">seeds.txtstring>
       <boolean name="reread-seeds-on-config">trueboolean>
       <newObject name="decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
         <map name="rules">
           <newObject name="rejectByDefault" class="org.archive.crawler.deciderules.RejectDecideRule">
           newObject>
           <newObject name="acceptIfSurtPrefixed" class="org.archive.crawler.deciderules.SurtPrefixedDecideRule">
             <string name="decision">ACCEPTstring>
             <string name="surts-source-file">string>
             <boolean name="seeds-as-surt-prefixes">trueboolean>
             <string name="surts-dump-file">string>
             <boolean name="also-check-via">falseboolean>
             <boolean name="rebuild-on-reconfig">trueboolean>
           newObject>
           <newObject name="rejectIfTooManyHops" class="org.archive.crawler.deciderules.TooManyHopsDecideRule">
             <integer name="max-hops">20integer>
           newObject>
           <newObject name="acceptIfTranscluded" class="org.archive.crawler.deciderules.TransclusionDecideRule">
             <integer name="max-trans-hops">3integer>
             <integer name="max-speculative-hops">1integer>
           newObject>
           <newObject name="rejectIfPathological" class="org.archive.crawler.deciderules.PathologicalPathDecideRule">
             <integer name="max-repetitions">2integer>
           newObject>
           <newObject name="rejectIfTooManyPathSegs" class="org.archive.crawler.deciderules.TooManyPathSegmentsDecideRule">
             <integer name="max-path-depth">20integer>
           newObject>
           <newObject name="acceptIfPrerequisite" class="org.archive.crawler.deciderules.PrerequisiteAcceptDecideRule">
           newObject>
         map>
       newObject>
     newObject>

 

    3.2： HTTP协议

   

 <map name="http-headers">
       <string name="user-agent">Mozilla/5.0 (compatible; heritrix/1.14.3 +http://127.0.0.1)string>
       <string name="from">[email protected]string>
     map>

 

    3.3：爬虫协议

    

 <newObject name="robots-honoring-policy" class="org.archive.crawler.datamodel.RobotsHonoringPolicy">
       <string name="type">classicstring>
       <boolean name="masquerade">falseboolean>
       <text name="custom-robots">text>
       <stringList name="user-agents">
       stringList>
     newObject>

 

   3.4：Frontier 调度器

  

 <newObject name="frontier" class="org.archive.crawler.frontier.BdbFrontier">
       <float name="delay-factor">4.0float>
       <integer name="max-delay-ms">20000integer>
       <integer name="min-delay-ms">2000integer>
       <integer name="respect-crawl-delay-up-to-secs">300integer>
       <integer name="max-retries">30integer>
       <long name="retry-delay-seconds">900long>
       <integer name="preference-embed-hops">1integer>
       <integer name="total-bandwidth-usage-KB-sec">0integer>
       <integer name="max-per-host-bandwidth-usage-KB-sec">0integer>
       <string name="queue-assignment-policy">org.archive.crawler.frontier.HostnameQueueAssignmentPolicystring>
       <string name="force-queue-assignment">string>
       <boolean name="pause-at-start">falseboolean>
       <boolean name="pause-at-finish">falseboolean>
       <boolean name="source-tag-seeds">falseboolean>
       <boolean name="recovery-log-enabled">trueboolean>
       <boolean name="hold-queues">trueboolean>
       <integer name="balance-replenish-amount">3000integer>
       <integer name="error-penalty-amount">100integer>
       <long name="queue-total-budget">-1long>
       <string name="cost-policy">org.archive.crawler.frontier.ZeroCostAssignmentPolicystring>
       <long name="snooze-deactivate-ms">300000long>
       <integer name="target-ready-backlog">50integer>
       <string name="uri-included-structure">org.archive.crawler.util.BdbUriUniqFilterstring>
       <boolean name="dump-pending-at-close">falseboolean>
     newObject>

 

   3.5：URL规范化规则，主要用来规范化每个URL，用Heritrix默认的就好了，这里不做说明了，其实也是通过各种规则

   3.6：预先处理链组件:

 <map name="pre-fetch-processors">
       <newObject name="Preselector" class="org.archive.crawler.prefetch.Preselector">
         <boolean name="enabled">trueboolean>
         <newObject name="Preselector#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
           <map name="rules">
           map>
         newObject>
         <boolean name="override-logger">falseboolean>
         <boolean name="recheck-scope">trueboolean>
         <boolean name="block-all">falseboolean>
         <string name="block-by-regexp">string>
         <string name="allow-by-regexp">string>
       newObject>
       <newObject name="Preprocessor" class="org.archive.crawler.prefetch.PreconditionEnforcer">
         <boolean name="enabled">trueboolean>
         <newObject name="Preprocessor#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
           <map name="rules">
           map>
         newObject>
         <integer name="ip-validity-duration-seconds">86400integer>
         <integer name="robot-validity-duration-seconds">86400integer>
         <boolean name="calculate-robots-only">falseboolean>
       newObject>
     map>

 

   3.7：获取组件:

 <map name="fetch-processors">
      <newObject name="DNS" class="org.archive.crawler.fetcher.FetchDNS">
        <boolean name="enabled">trueboolean>
        <newObject name="DNS#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
          <map name="rules">
          map>
        newObject>
        <boolean name="accept-non-dns-resolves">falseboolean>
        <boolean name="digest-content">trueboolean>
        <string name="digest-algorithm">sha1string>
      newObject>
      <newObject name="HTTP" class="org.archive.crawler.fetcher.FetchHTTP">
        <boolean name="enabled">trueboolean>
        <newObject name="HTTP#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
          <map name="rules">
          map>
        newObject>
        <newObject name="midfetch-decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
          <map name="rules">
          map>
        newObject>
        <integer name="timeout-seconds">1200integer>
        <integer name="sotimeout-ms">20000integer>
        <integer name="fetch-bandwidth">0integer>
        <long name="max-length-bytes">0long>
        <boolean name="ignore-cookies">falseboolean>
        <boolean name="use-bdb-for-cookies">trueboolean>
        <string name="load-cookies-from-file">string>
        <string name="save-cookies-to-file">string>
        <string name="trust-level">openstring>
        <stringList name="accept-headers">
        stringList>
        <string name="http-proxy-host">string>
        <string name="http-proxy-port">string>
        <string name="default-encoding">GB2312string>
        <boolean name="digest-content">trueboolean>
        <string name="digest-algorithm">sha1string>
        <boolean name="send-if-modified-since">trueboolean>
        <boolean name="send-if-none-match">trueboolean>
        <boolean name="send-connection-close">trueboolean>
        <boolean name="send-referer">trueboolean>
        <boolean name="send-range">falseboolean>
        <string name="http-bind-address">string>
      newObject>
    map>

 

   3.8：抽取组件

 <map name="extract-processors">
      <newObject name="ExtractorHTTP" class="org.archive.crawler.extractor.ExtractorHTTP">
        <boolean name="enabled">trueboolean>
        <newObject name="ExtractorHTTP#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
          <map name="rules">
          map>
        newObject>
      newObject>
      <newObject name="ExtractorHTML" class="org.archive.crawler.extractor.ExtractorHTML">
        <boolean name="enabled">trueboolean>
        <newObject name="ExtractorHTML#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
          <map name="rules">
          map>
        newObject>
        <boolean name="extract-javascript">trueboolean>
        <boolean name="treat-frames-as-embed-links">trueboolean>
        <boolean name="ignore-form-action-urls">trueboolean>
        <boolean name="extract-only-form-gets">trueboolean>
        <boolean name="extract-value-attributes">trueboolean>
        <boolean name="ignore-unexpected-html">trueboolean>
      newObject>
    map>

 

   3.9：写组件

 <map name="write-processors">
       <newObject name="Archiver" class="com.steel.heritrix.extend.MyWriterMirror">
         <boolean name="enabled">trueboolean>
         <newObject name="Archiver#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
           <map name="rules">
           map>
         newObject>
         <boolean name="case-sensitive">trueboolean>
         <stringList name="character-map" />
         <stringList name="content-type-map" />
         <string name="directory-file">index.htmlstring>
         <string name="dot-begin">%2Estring>
         <string name="dot-end">.string>
         <stringList name="host-map" />
         <boolean name="host-directory">trueboolean>
         <string name="path">mirrorstring>
         <integer name="max-path-length">1023integer>
         <integer name="max-segment-length">255integer>
         <boolean name="port-directory">falseboolean>
         <boolean name="suffix-at-end">trueboolean>
         <string name="too-long-directory">LONGstring>
         <stringList name="underscore-set" />
       newObject>
     map>

 

   3.10：请求链组件

 <map name="post-processors">
       <newObject name="Updater" class="org.archive.crawler.postprocessor.CrawlStateUpdater">
         <boolean name="enabled">trueboolean>
         <newObject name="Updater#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
           <map name="rules">
           map>
         newObject>
       newObject>
       <newObject name="LinksScoper" class="org.archive.crawler.postprocessor.LinksScoper">
         <boolean name="enabled">trueboolean>
         <newObject name="LinksScoper#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
           <map name="rules">
           map>
         newObject>
         <boolean name="override-logger">falseboolean>
         <integer name="preference-depth-hops">-1integer>
         <newObject name="scope-rejected-url-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
           <map name="rules">
           map>
         newObject>
       newObject>
       <newObject name="Scheduler" class="com.steel.heritrix.extend.MyFrontierScheduler">
         <boolean name="enabled">trueboolean>
         <newObject name="Scheduler#decide-rules" class="org.archive.crawler.deciderules.DecideRuleSequence">
           <map name="rules">
           map>
         newObject>
       newObject>
     map>

 

   3.11：统计跟踪链组件

 <map name="loggers">
       <newObject name="crawl-statistics" class="org.archive.crawler.admin.StatisticsTracker">
         <integer name="interval-seconds">20integer>
       newObject>
     map>