Back to index

tor  0.2.3.19-rc
directory.c
Go to the documentation of this file.
00001 /* Copyright (c) 2001-2004, Roger Dingledine.
00002  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
00003  * Copyright (c) 2007-2012, The Tor Project, Inc. */
00004 /* See LICENSE for licensing information */
00005 
00006 #include "or.h"
00007 #include "buffers.h"
00008 #include "circuitbuild.h"
00009 #include "config.h"
00010 #include "connection.h"
00011 #include "connection_edge.h"
00012 #include "control.h"
00013 #include "directory.h"
00014 #include "dirserv.h"
00015 #include "dirvote.h"
00016 #include "geoip.h"
00017 #include "main.h"
00018 #include "microdesc.h"
00019 #include "networkstatus.h"
00020 #include "nodelist.h"
00021 #include "policies.h"
00022 #include "rendclient.h"
00023 #include "rendcommon.h"
00024 #include "rephist.h"
00025 #include "router.h"
00026 #include "routerlist.h"
00027 #include "routerparse.h"
00028 
00029 #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
00030 #ifndef OPENBSD
00031 #include <malloc.h>
00032 #endif
00033 #endif
00034 
00042 /* In-points to directory.c:
00043  *
00044  * - directory_post_to_dirservers(), called from
00045  *   router_upload_dir_desc_to_dirservers() in router.c
00046  *   upload_service_descriptor() in rendservice.c
00047  * - directory_get_from_dirserver(), called from
00048  *   rend_client_refetch_renddesc() in rendclient.c
00049  *   run_scheduled_events() in main.c
00050  *   do_hup() in main.c
00051  * - connection_dir_process_inbuf(), called from
00052  *   connection_process_inbuf() in connection.c
00053  * - connection_dir_finished_flushing(), called from
00054  *   connection_finished_flushing() in connection.c
00055  * - connection_dir_finished_connecting(), called from
00056  *   connection_finished_connecting() in connection.c
00057  */
00058 static void directory_send_command(dir_connection_t *conn,
00059                              int purpose, int direct, const char *resource,
00060                              const char *payload, size_t payload_len,
00061                              int supports_conditional_consensus,
00062                              time_t if_modified_since);
00063 static int directory_handle_command(dir_connection_t *conn);
00064 static int body_is_plausible(const char *body, size_t body_len, int purpose);
00065 static int purpose_needs_anonymity(uint8_t dir_purpose,
00066                                    uint8_t router_purpose);
00067 static char *http_get_header(const char *headers, const char *which);
00068 static void http_set_address_origin(const char *headers, connection_t *conn);
00069 static void connection_dir_download_v2_networkstatus_failed(
00070                                dir_connection_t *conn, int status_code);
00071 static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
00072 static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
00073 static void connection_dir_download_cert_failed(
00074                                dir_connection_t *conn, int status_code);
00075 static void connection_dir_retry_bridges(smartlist_t *descs);
00076 static void dir_networkstatus_download_failed(smartlist_t *failed,
00077                                               int status_code);
00078 static void dir_routerdesc_download_failed(smartlist_t *failed,
00079                                            int status_code,
00080                                            int router_purpose,
00081                                            int was_extrainfo,
00082                                            int was_descriptor_digests);
00083 static void dir_microdesc_download_failed(smartlist_t *failed,
00084                                           int status_code);
00085 static void note_client_request(int purpose, int compressed, size_t bytes);
00086 static int client_likes_consensus(networkstatus_t *v, const char *want_url);
00087 
00088 static void directory_initiate_command_rend(const char *address,
00089                                             const tor_addr_t *addr,
00090                                             uint16_t or_port,
00091                                             uint16_t dir_port,
00092                                             int supports_conditional_consensus,
00093                                             int supports_begindir,
00094                                             const char *digest,
00095                                             uint8_t dir_purpose,
00096                                             uint8_t router_purpose,
00097                                             int anonymized_connection,
00098                                             const char *resource,
00099                                             const char *payload,
00100                                             size_t payload_len,
00101                                             time_t if_modified_since,
00102                                             const rend_data_t *rend_query);
00103 
00104 /********* START VARIABLES **********/
00105 
00108 #define ALLOW_DIRECTORY_TIME_SKEW (30*60)
00109 
00110 #define X_ADDRESS_HEADER "X-Your-Address-Is: "
00111 
00114 #define FULL_DIR_CACHE_LIFETIME (60*60)
00115 #define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
00116 #define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
00117 #define NETWORKSTATUS_CACHE_LIFETIME (5*60)
00118 #define ROUTERDESC_CACHE_LIFETIME (30*60)
00119 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
00120 #define ROBOTS_CACHE_LIFETIME (24*60*60)
00121 #define MICRODESC_CACHE_LIFETIME (48*60*60)
00122 
00123 /********* END VARIABLES ************/
00124 
00128 static int
00129 purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
00130 {
00131   if (get_options()->AllDirActionsPrivate)
00132     return 1;
00133   if (router_purpose == ROUTER_PURPOSE_BRIDGE)
00134     return 1; /* if no circuits yet, this might break bootstrapping, but it's
00135                * needed to be safe. */
00136   if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
00137       dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
00138       dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES ||
00139       dir_purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS ||
00140       dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
00141       dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES ||
00142       dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
00143       dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
00144       dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
00145       dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
00146       dir_purpose == DIR_PURPOSE_FETCH_MICRODESC)
00147     return 0;
00148   return 1;
00149 }
00150 
00153 static char *
00154 authdir_type_to_string(dirinfo_type_t auth)
00155 {
00156   char *result;
00157   smartlist_t *lst = smartlist_new();
00158   if (auth & V1_DIRINFO)
00159     smartlist_add(lst, (void*)"V1");
00160   if (auth & V2_DIRINFO)
00161     smartlist_add(lst, (void*)"V2");
00162   if (auth & V3_DIRINFO)
00163     smartlist_add(lst, (void*)"V3");
00164   if (auth & BRIDGE_DIRINFO)
00165     smartlist_add(lst, (void*)"Bridge");
00166   if (auth & HIDSERV_DIRINFO)
00167     smartlist_add(lst, (void*)"Hidden service");
00168   if (smartlist_len(lst)) {
00169     result = smartlist_join_strings(lst, ", ", 0, NULL);
00170   } else {
00171     result = tor_strdup("[Not an authority]");
00172   }
00173   smartlist_free(lst);
00174   return result;
00175 }
00176 
00178 static const char *
00179 dir_conn_purpose_to_string(int purpose)
00180 {
00181   switch (purpose)
00182     {
00183     case DIR_PURPOSE_FETCH_RENDDESC:
00184       return "hidden-service descriptor fetch";
00185     case DIR_PURPOSE_UPLOAD_DIR:
00186       return "server descriptor upload";
00187     case DIR_PURPOSE_UPLOAD_RENDDESC:
00188       return "hidden-service descriptor upload";
00189     case DIR_PURPOSE_UPLOAD_VOTE:
00190       return "server vote upload";
00191     case DIR_PURPOSE_UPLOAD_SIGNATURES:
00192       return "consensus signature upload";
00193     case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS:
00194       return "network-status fetch";
00195     case DIR_PURPOSE_FETCH_SERVERDESC:
00196       return "server descriptor fetch";
00197     case DIR_PURPOSE_FETCH_EXTRAINFO:
00198       return "extra-info fetch";
00199     case DIR_PURPOSE_FETCH_CONSENSUS:
00200       return "consensus network-status fetch";
00201     case DIR_PURPOSE_FETCH_CERTIFICATE:
00202       return "authority cert fetch";
00203     case DIR_PURPOSE_FETCH_STATUS_VOTE:
00204       return "status vote fetch";
00205     case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
00206       return "consensus signature fetch";
00207     case DIR_PURPOSE_FETCH_RENDDESC_V2:
00208       return "hidden-service v2 descriptor fetch";
00209     case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
00210       return "hidden-service v2 descriptor upload";
00211     case DIR_PURPOSE_FETCH_MICRODESC:
00212       return "microdescriptor fetch";
00213     }
00214 
00215   log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
00216   return "(unknown)";
00217 }
00218 
00222 int
00223 router_supports_extrainfo(const char *identity_digest, int is_authority)
00224 {
00225   const node_t *node = node_get_by_id(identity_digest);
00226 
00227   if (node && node->ri) {
00228     if (node->ri->caches_extra_info)
00229       return 1;
00230     if (is_authority && node->ri->platform &&
00231         tor_version_as_new_as(node->ri->platform,
00232                               "Tor 0.2.0.0-alpha-dev (r10070)"))
00233       return 1;
00234   }
00235   if (is_authority) {
00236     const routerstatus_t *rs =
00237       router_get_consensus_status_by_id(identity_digest);
00238     if (rs && rs->version_supports_extrainfo_upload)
00239       return 1;
00240   }
00241   return 0;
00242 }
00243 
00252 int
00253 directories_have_accepted_server_descriptor(void)
00254 {
00255   smartlist_t *servers = router_get_trusted_dir_servers();
00256   const or_options_t *options = get_options();
00257   SMARTLIST_FOREACH(servers, trusted_dir_server_t *, d, {
00258     if ((d->type & options->_PublishServerDescriptor) &&
00259         d->has_accepted_serverdesc) {
00260       return 1;
00261     }
00262   });
00263   return 0;
00264 }
00265 
00283 void
00284 directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
00285                              dirinfo_type_t type,
00286                              const char *payload,
00287                              size_t payload_len, size_t extrainfo_len)
00288 {
00289   const or_options_t *options = get_options();
00290   int post_via_tor;
00291   smartlist_t *dirservers = router_get_trusted_dir_servers();
00292   int found = 0;
00293   const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
00294                             dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
00295   tor_assert(dirservers);
00296   /* This tries dirservers which we believe to be down, but ultimately, that's
00297    * harmless, and we may as well err on the side of getting things uploaded.
00298    */
00299   SMARTLIST_FOREACH_BEGIN(dirservers, trusted_dir_server_t *, ds) {
00300       routerstatus_t *rs = &(ds->fake_status);
00301       size_t upload_len = payload_len;
00302       tor_addr_t ds_addr;
00303 
00304       if ((type & ds->type) == 0)
00305         continue;
00306 
00307       if (exclude_self && router_digest_is_me(ds->digest))
00308         continue;
00309 
00310       if (options->StrictNodes &&
00311           routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) {
00312         log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but "
00313                  "it's in our ExcludedNodes list and StrictNodes is set. "
00314                  "Skipping.",
00315                  ds->nickname,
00316                  dir_conn_purpose_to_string(dir_purpose));
00317         continue;
00318       }
00319 
00320       found = 1; /* at least one authority of this type was listed */
00321       if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
00322         ds->has_accepted_serverdesc = 0;
00323 
00324       if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
00325         upload_len += extrainfo_len;
00326         log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
00327                  (int) extrainfo_len);
00328       }
00329       tor_addr_from_ipv4h(&ds_addr, ds->addr);
00330       post_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose) ||
00331         !fascist_firewall_allows_address_dir(&ds_addr, ds->dir_port);
00332       directory_initiate_command_routerstatus(rs, dir_purpose,
00333                                               router_purpose,
00334                                               post_via_tor,
00335                                               NULL, payload, upload_len, 0);
00336   } SMARTLIST_FOREACH_END(ds);
00337   if (!found) {
00338     char *s = authdir_type_to_string(type);
00339     log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
00340              "of type '%s', but no authorities of that type listed!", s);
00341     tor_free(s);
00342   }
00343 }
00344 
00351 void
00352 directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
00353                              const char *resource, int pds_flags)
00354 {
00355   const routerstatus_t *rs = NULL;
00356   const or_options_t *options = get_options();
00357   int prefer_authority = directory_fetches_from_authorities(options);
00358   int require_authority = 0;
00359   int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose);
00360   dirinfo_type_t type;
00361   time_t if_modified_since = 0;
00362 
00363   /* FFFF we could break this switch into its own function, and call
00364    * it elsewhere in directory.c. -RD */
00365   switch (dir_purpose) {
00366     case DIR_PURPOSE_FETCH_EXTRAINFO:
00367       type = EXTRAINFO_DIRINFO |
00368              (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO :
00369                                                         V3_DIRINFO);
00370       break;
00371     case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS:
00372       type = V2_DIRINFO;
00373       prefer_authority = 1; /* Only v2 authorities have these anyway. */
00374       require_authority = 1; /* Don't fallback to asking a non-authority */
00375       break;
00376     case DIR_PURPOSE_FETCH_SERVERDESC:
00377       type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO :
00378                                                         V3_DIRINFO);
00379       break;
00380     case DIR_PURPOSE_FETCH_RENDDESC:
00381       type = HIDSERV_DIRINFO;
00382       break;
00383     case DIR_PURPOSE_FETCH_STATUS_VOTE:
00384     case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
00385     case DIR_PURPOSE_FETCH_CERTIFICATE:
00386       type = V3_DIRINFO;
00387       break;
00388     case DIR_PURPOSE_FETCH_CONSENSUS:
00389       type = V3_DIRINFO;
00390       if (resource && !strcmp(resource,"microdesc"))
00391         type |= MICRODESC_DIRINFO;
00392       break;
00393     case DIR_PURPOSE_FETCH_MICRODESC:
00394       type = MICRODESC_DIRINFO;
00395       break;
00396     default:
00397       log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
00398       return;
00399   }
00400 
00401   if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
00402     int flav = FLAV_NS;
00403     networkstatus_t *v;
00404     if (resource)
00405       flav = networkstatus_parse_flavor_name(resource);
00406 
00407     if (flav != -1) {
00408       /* IF we have a parsed consensus of this type, we can do an
00409        * if-modified-time based on it. */
00410       v = networkstatus_get_latest_consensus_by_flavor(flav);
00411       if (v)
00412         if_modified_since = v->valid_after + 180;
00413     } else {
00414       /* Otherwise it might be a consensus we don't parse, but which we
00415        * do cache.  Look at the cached copy, perhaps. */
00416       cached_dir_t *cd = dirserv_get_consensus(resource);
00417       if (cd)
00418         if_modified_since = cd->published + 180;
00419     }
00420   }
00421 
00422   if (!options->FetchServerDescriptors && type != HIDSERV_DIRINFO)
00423     return;
00424 
00425   if (!get_via_tor) {
00426     if (options->UseBridges && type != BRIDGE_DIRINFO) {
00427       /* We want to ask a running bridge for which we have a descriptor.
00428        *
00429        * Be careful here: we should only ask questions that we know our
00430        * bridges can answer. So far we're solving that by backing off to
00431        * the behavior supported by our oldest bridge; see for example
00432        * any_bridges_dont_support_microdescriptors().
00433        */
00434       /* XXX024 Not all bridges handle conditional consensus downloading,
00435        * so, for now, never assume the server supports that. -PP */
00436       const node_t *node = choose_random_entry(NULL);
00437       if (node && node->ri) {
00438         /* every bridge has a routerinfo. */
00439         tor_addr_t addr;
00440         routerinfo_t *ri = node->ri;
00441         node_get_addr(node, &addr);
00442         directory_initiate_command(ri->address, &addr,
00443                                    ri->or_port, 0,
00444                                    0, /* don't use conditional consensus url */
00445                                    1, ri->cache_info.identity_digest,
00446                                    dir_purpose,
00447                                    router_purpose,
00448                                    0, resource, NULL, 0, if_modified_since);
00449       } else
00450         log_notice(LD_DIR, "Ignoring directory request, since no bridge "
00451                            "nodes are available yet.");
00452       return;
00453     } else {
00454       if (prefer_authority || type == BRIDGE_DIRINFO) {
00455         /* only ask authdirservers, and don't ask myself */
00456         rs = router_pick_trusteddirserver(type, pds_flags);
00457         if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
00458                                         PDS_NO_EXISTING_MICRODESC_FETCH))) {
00459           /* We don't want to fetch from any authorities that we're currently
00460            * fetching server descriptors from, and we got no match.  Did we
00461            * get no match because all the authorities have connections
00462            * fetching server descriptors (in which case we should just
00463            * return,) or because all the authorities are down or on fire or
00464            * unreachable or something (in which case we should go on with
00465            * our fallback code)? */
00466           pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
00467                          PDS_NO_EXISTING_MICRODESC_FETCH);
00468           rs = router_pick_trusteddirserver(type, pds_flags);
00469           if (rs) {
00470             log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
00471                       "are in use.");
00472             return;
00473           }
00474         }
00475         if (rs == NULL && require_authority) {
00476           log_info(LD_DIR, "No authorities were available for %s: will try "
00477                    "later.", dir_conn_purpose_to_string(dir_purpose));
00478           return;
00479         }
00480       }
00481       if (!rs && type != BRIDGE_DIRINFO) {
00482         /* anybody with a non-zero dirport will do */
00483         rs = router_pick_directory_server(type, pds_flags);
00484         if (!rs) {
00485           log_info(LD_DIR, "No router found for %s; falling back to "
00486                    "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
00487           rs = router_pick_trusteddirserver(type, pds_flags);
00488           if (!rs)
00489             get_via_tor = 1; /* last resort: try routing it via Tor */
00490         }
00491       }
00492     }
00493   } else { /* get_via_tor */
00494     /* Never use fascistfirewall; we're going via Tor. */
00495     if (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) {
00496       /* only ask hidserv authorities, any of them will do */
00497       pds_flags |= PDS_IGNORE_FASCISTFIREWALL|PDS_ALLOW_SELF;
00498       rs = router_pick_trusteddirserver(HIDSERV_DIRINFO, pds_flags);
00499     } else {
00500       /* anybody with a non-zero dirport will do. Disregard firewalls. */
00501       pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
00502       rs = router_pick_directory_server(type, pds_flags);
00503       /* If we have any hope of building an indirect conn, we know some router
00504        * descriptors.  If (rs==NULL), we can't build circuits anyway, so
00505        * there's no point in falling back to the authorities in this case. */
00506     }
00507   }
00508 
00509   if (rs)
00510     directory_initiate_command_routerstatus(rs, dir_purpose,
00511                                             router_purpose,
00512                                             get_via_tor,
00513                                             resource, NULL, 0,
00514                                             if_modified_since);
00515   else {
00516     log_notice(LD_DIR,
00517                "While fetching directory info, "
00518                "no running dirservers known. Will try again later. "
00519                "(purpose %d)", dir_purpose);
00520     if (!purpose_needs_anonymity(dir_purpose, router_purpose)) {
00521       /* remember we tried them all and failed. */
00522       directory_all_unreachable(time(NULL));
00523     }
00524   }
00525 }
00526 
00530 void
00531 directory_get_from_all_authorities(uint8_t dir_purpose,
00532                                    uint8_t router_purpose,
00533                                    const char *resource)
00534 {
00535   tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
00536              dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
00537 
00538   SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
00539                     trusted_dir_server_t *, ds,
00540     {
00541       routerstatus_t *rs;
00542       if (router_digest_is_me(ds->digest))
00543         continue;
00544       if (!(ds->type & V3_DIRINFO))
00545         continue;
00546       rs = &ds->fake_status;
00547       directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose,
00548                                               0, resource, NULL, 0, 0);
00549     });
00550 }
00551 
00554 void
00555 directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
00556                                              uint8_t dir_purpose,
00557                                              uint8_t router_purpose,
00558                                              int anonymized_connection,
00559                                              const char *resource,
00560                                              const char *payload,
00561                                              size_t payload_len,
00562                                              time_t if_modified_since,
00563                                              const rend_data_t *rend_query)
00564 {
00565   const or_options_t *options = get_options();
00566   const node_t *node;
00567   char address_buf[INET_NTOA_BUF_LEN+1];
00568   struct in_addr in;
00569   const char *address;
00570   tor_addr_t addr;
00571   node = node_get_by_id(status->identity_digest);
00572 
00573   if (!node && anonymized_connection) {
00574     log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
00575              "don't have its router descriptor.",
00576              routerstatus_describe(status));
00577     return;
00578   } else if (node) {
00579     node_get_address_string(node, address_buf, sizeof(address_buf));
00580     address = address_buf;
00581   } else {
00582     in.s_addr = htonl(status->addr);
00583     tor_inet_ntoa(&in, address_buf, sizeof(address_buf));
00584     address = address_buf;
00585   }
00586   tor_addr_from_ipv4h(&addr, status->addr);
00587 
00588   if (options->ExcludeNodes && options->StrictNodes &&
00589       routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
00590     log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
00591              "it's in our ExcludedNodes list and StrictNodes is set. "
00592              "Skipping. This choice might make your Tor not work.",
00593              routerstatus_describe(status),
00594              dir_conn_purpose_to_string(dir_purpose));
00595     return;
00596   }
00597 
00598   directory_initiate_command_rend(address, &addr,
00599                              status->or_port, status->dir_port,
00600                              status->version_supports_conditional_consensus,
00601                              status->version_supports_begindir,
00602                              status->identity_digest,
00603                              dir_purpose, router_purpose,
00604                              anonymized_connection, resource,
00605                              payload, payload_len, if_modified_since,
00606                              rend_query);
00607 }
00608 
00623 void
00624 directory_initiate_command_routerstatus(const routerstatus_t *status,
00625                                         uint8_t dir_purpose,
00626                                         uint8_t router_purpose,
00627                                         int anonymized_connection,
00628                                         const char *resource,
00629                                         const char *payload,
00630                                         size_t payload_len,
00631                                         time_t if_modified_since)
00632 {
00633   directory_initiate_command_routerstatus_rend(status, dir_purpose,
00634                                           router_purpose,
00635                                           anonymized_connection, resource,
00636                                           payload, payload_len,
00637                                           if_modified_since, NULL);
00638 }
00639 
00643 static int
00644 directory_conn_is_self_reachability_test(dir_connection_t *conn)
00645 {
00646   if (conn->requested_resource &&
00647       !strcmpstart(conn->requested_resource,"authority")) {
00648     const routerinfo_t *me = router_get_my_routerinfo();
00649     if (me &&
00650         router_digest_is_me(conn->identity_digest) &&
00651         tor_addr_eq_ipv4h(&conn->_base.addr, me->addr) && /*XXXX prop 118*/
00652         me->dir_port == conn->_base.port)
00653       return 1;
00654   }
00655   return 0;
00656 }
00657 
00662 static void
00663 connection_dir_request_failed(dir_connection_t *conn)
00664 {
00665   if (directory_conn_is_self_reachability_test(conn)) {
00666     return; /* this was a test fetch. don't retry. */
00667   }
00668   if (!entry_list_is_constrained(get_options()))
00669     router_set_status(conn->identity_digest, 0); /* don't try him again */
00670   if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) {
00671     log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
00672              conn->_base.address);
00673     connection_dir_download_v2_networkstatus_failed(conn, -1);
00674   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
00675              conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
00676     log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
00677              "directory server at '%s'; retrying",
00678              conn->_base.address);
00679     if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
00680       connection_dir_bridge_routerdesc_failed(conn);
00681     connection_dir_download_routerdesc_failed(conn);
00682   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
00683     if (conn->requested_resource)
00684       networkstatus_consensus_download_failed(0, conn->requested_resource);
00685   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
00686     log_info(LD_DIR, "Giving up on certificate fetch from directory server "
00687              "at '%s'; retrying",
00688              conn->_base.address);
00689     connection_dir_download_cert_failed(conn, 0);
00690   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
00691     log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
00692              conn->_base.address);
00693   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
00694     log_info(LD_DIR, "Giving up downloading votes from '%s'",
00695              conn->_base.address);
00696   } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
00697     log_info(LD_DIR, "Giving up on downloading microdescriptors from "
00698              " directory server at '%s'; will retry", conn->_base.address);
00699     connection_dir_download_routerdesc_failed(conn);
00700   }
00701 }
00702 
00707 static void
00708 connection_dir_download_v2_networkstatus_failed(dir_connection_t *conn,
00709                                              int status_code)
00710 {
00711   if (!conn->requested_resource) {
00712     /* We never reached directory_send_command, which means that we never
00713      * opened a network connection.  Either we're out of sockets, or the
00714      * network is down.  Either way, retrying would be pointless. */
00715     return;
00716   }
00717   if (!strcmpstart(conn->requested_resource, "all")) {
00718     /* We're a non-authoritative directory cache; try again. Ignore status
00719      * code, since we don't want to keep trying forever in a tight loop
00720      * if all the authorities are shutting us out. */
00721     smartlist_t *trusted_dirs = router_get_trusted_dir_servers();
00722     SMARTLIST_FOREACH(trusted_dirs, trusted_dir_server_t *, ds,
00723                       download_status_failed(&ds->v2_ns_dl_status, 0));
00724     directory_get_from_dirserver(conn->_base.purpose, conn->router_purpose,
00725                                  "all.z", 0 /* don't retry_if_no_servers */);
00726   } else if (!strcmpstart(conn->requested_resource, "fp/")) {
00727     /* We were trying to download by fingerprint; mark them all as having
00728      * failed, and possibly retry them later.*/
00729     smartlist_t *failed = smartlist_new();
00730     dir_split_resource_into_fingerprints(conn->requested_resource+3,
00731                                          failed, NULL, 0);
00732     if (smartlist_len(failed)) {
00733       dir_networkstatus_download_failed(failed, status_code);
00734       SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp));
00735     }
00736     smartlist_free(failed);
00737   }
00738 }
00739 
00743 static void
00744 connection_dir_retry_bridges(smartlist_t *descs)
00745 {
00746   char digest[DIGEST_LEN];
00747   SMARTLIST_FOREACH(descs, const char *, cp,
00748   {
00749     if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) {
00750       log_warn(LD_BUG, "Malformed fingerprint in list: %s",
00751               escaped(cp));
00752       continue;
00753     }
00754     retry_bridge_descriptor_fetch_directly(digest);
00755   });
00756 }
00757 
00761 static void
00762 connection_dir_download_routerdesc_failed(dir_connection_t *conn)
00763 {
00764   /* No need to increment the failure count for routerdescs, since
00765    * it's not their fault. */
00766 
00767   /* No need to relaunch descriptor downloads here: we already do it
00768    * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
00769   tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
00770              conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
00771              conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);
00772 
00773   (void) conn;
00774 }
00775 
00780 static void
00781 connection_dir_bridge_routerdesc_failed(dir_connection_t *conn)
00782 {
00783   smartlist_t *which = NULL;
00784 
00785   /* Requests for bridge descriptors are in the form 'fp/', so ignore
00786      anything else. */
00787   if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/"))
00788     return;
00789 
00790   which = smartlist_new();
00791   dir_split_resource_into_fingerprints(conn->requested_resource
00792                                         + strlen("fp/"),
00793                                        which, NULL, 0);
00794 
00795   tor_assert(conn->_base.purpose != DIR_PURPOSE_FETCH_EXTRAINFO);
00796   if (smartlist_len(which)) {
00797     connection_dir_retry_bridges(which);
00798     SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
00799   }
00800   smartlist_free(which);
00801 }
00802 
00804 static void
00805 connection_dir_download_cert_failed(dir_connection_t *conn, int status)
00806 {
00807   smartlist_t *failed;
00808   tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
00809 
00810   if (!conn->requested_resource)
00811     return;
00812   failed = smartlist_new();
00813   dir_split_resource_into_fingerprints(conn->requested_resource+3,
00814                                        failed, NULL, DSR_HEX);
00815   SMARTLIST_FOREACH(failed, char *, cp,
00816   {
00817     authority_cert_dl_failed(cp, status);
00818     tor_free(cp);
00819   });
00820   smartlist_free(failed);
00821 
00822   update_certificate_downloads(time(NULL));
00823 }
00824 
00833 static int
00834 directory_command_should_use_begindir(const or_options_t *options,
00835                                       const tor_addr_t *addr,
00836                                       int or_port, uint8_t router_purpose,
00837                                       int anonymized_connection)
00838 {
00839   if (!or_port)
00840     return 0; /* We don't know an ORPort -- no chance. */
00841   if (!anonymized_connection)
00842     if (!fascist_firewall_allows_address_or(addr, or_port) ||
00843         directory_fetches_from_authorities(options))
00844       return 0; /* We're firewalled or are acting like a relay -- also no. */
00845   if (!options->TunnelDirConns &&
00846       router_purpose != ROUTER_PURPOSE_BRIDGE)
00847     return 0; /* We prefer to avoid using begindir conns. Fine. */
00848   return 1;
00849 }
00850 
00856 void
00857 directory_initiate_command(const char *address, const tor_addr_t *_addr,
00858                            uint16_t or_port, uint16_t dir_port,
00859                            int supports_conditional_consensus,
00860                            int supports_begindir, const char *digest,
00861                            uint8_t dir_purpose, uint8_t router_purpose,
00862                            int anonymized_connection, const char *resource,
00863                            const char *payload, size_t payload_len,
00864                            time_t if_modified_since)
00865 {
00866   directory_initiate_command_rend(address, _addr, or_port, dir_port,
00867                              supports_conditional_consensus,
00868                              supports_begindir, digest, dir_purpose,
00869                              router_purpose, anonymized_connection,
00870                              resource, payload, payload_len,
00871                              if_modified_since, NULL);
00872 }
00873 
00878 static int
00879 is_sensitive_dir_purpose(uint8_t dir_purpose)
00880 {
00881   return ((dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) ||
00882           (dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC) ||
00883           (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC) ||
00884           (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) ||
00885           (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2));
00886 }
00887 
00890 static void
00891 directory_initiate_command_rend(const char *address, const tor_addr_t *_addr,
00892                                 uint16_t or_port, uint16_t dir_port,
00893                                 int supports_conditional_consensus,
00894                                 int supports_begindir, const char *digest,
00895                                 uint8_t dir_purpose, uint8_t router_purpose,
00896                                 int anonymized_connection,
00897                                 const char *resource,
00898                                 const char *payload, size_t payload_len,
00899                                 time_t if_modified_since,
00900                                 const rend_data_t *rend_query)
00901 {
00902   dir_connection_t *conn;
00903   const or_options_t *options = get_options();
00904   int socket_error = 0;
00905   int use_begindir = supports_begindir &&
00906                      directory_command_should_use_begindir(options, _addr,
00907                        or_port, router_purpose, anonymized_connection);
00908   tor_addr_t addr;
00909 
00910   tor_assert(address);
00911   tor_assert(_addr);
00912   tor_assert(or_port || dir_port);
00913   tor_assert(digest);
00914 
00915   tor_addr_copy(&addr, _addr);
00916 
00917   log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
00918             anonymized_connection, use_begindir);
00919 
00920   log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
00921 
00922 #ifndef NON_ANONYMOUS_MODE_ENABLED
00923   tor_assert(!(is_sensitive_dir_purpose(dir_purpose) &&
00924                !anonymized_connection));
00925 #else
00926   (void)is_sensitive_dir_purpose;
00927 #endif
00928 
00929   /* ensure that we don't make direct connections when a SOCKS server is
00930    * configured. */
00931   if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
00932       (options->Socks4Proxy || options->Socks5Proxy)) {
00933     log_warn(LD_DIR, "Cannot connect to a directory server through a "
00934                      "SOCKS proxy!");
00935     return;
00936   }
00937 
00938   conn = dir_connection_new(tor_addr_family(&addr));
00939 
00940   /* set up conn so it's got all the data we need to remember */
00941   tor_addr_copy(&conn->_base.addr, &addr);
00942   conn->_base.port = use_begindir ? or_port : dir_port;
00943   conn->_base.address = tor_strdup(address);
00944   memcpy(conn->identity_digest, digest, DIGEST_LEN);
00945 
00946   conn->_base.purpose = dir_purpose;
00947   conn->router_purpose = router_purpose;
00948 
00949   /* give it an initial state */
00950   conn->_base.state = DIR_CONN_STATE_CONNECTING;
00951 
00952   /* decide whether we can learn our IP address from this conn */
00953   conn->dirconn_direct = !anonymized_connection;
00954 
00955   /* copy rendezvous data, if any */
00956   if (rend_query)
00957     conn->rend_data = rend_data_dup(rend_query);
00958 
00959   if (!anonymized_connection && !use_begindir) {
00960     /* then we want to connect to dirport directly */
00961 
00962     if (options->HTTPProxy) {
00963       tor_addr_copy(&addr, &options->HTTPProxyAddr);
00964       dir_port = options->HTTPProxyPort;
00965     }
00966 
00967     switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr,
00968                                dir_port, &socket_error)) {
00969       case -1:
00970         connection_dir_request_failed(conn); /* retry if we want */
00971         /* XXX we only pass 'conn' above, not 'resource', 'payload',
00972          * etc. So in many situations it can't retry! -RD */
00973         connection_free(TO_CONN(conn));
00974         return;
00975       case 1:
00976         /* start flushing conn */
00977         conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
00978         /* fall through */
00979       case 0:
00980         /* queue the command on the outbuf */
00981         directory_send_command(conn, dir_purpose, 1, resource,
00982                                payload, payload_len,
00983                                supports_conditional_consensus,
00984                                if_modified_since);
00985         connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
00986         /* writable indicates finish, readable indicates broken link,
00987            error indicates broken link in windowsland. */
00988     }
00989   } else { /* we want to connect via a tor connection */
00990     entry_connection_t *linked_conn;
00991     /* Anonymized tunneled connections can never share a circuit.
00992      * One-hop directory connections can share circuits with each other
00993      * but nothing else. */
00994     int iso_flags = anonymized_connection ? ISO_STREAM : ISO_SESSIONGRP;
00995 
00996     /* If it's an anonymized connection, remember the fact that we
00997      * wanted it for later: maybe we'll want it again soon. */
00998     if (anonymized_connection && use_begindir)
00999       rep_hist_note_used_internal(time(NULL), 0, 1);
01000     else if (anonymized_connection && !use_begindir)
01001       rep_hist_note_used_port(time(NULL), conn->_base.port);
01002 
01003     /* make an AP connection
01004      * populate it and add it at the right state
01005      * hook up both sides
01006      */
01007     linked_conn =
01008       connection_ap_make_link(TO_CONN(conn),
01009                               conn->_base.address, conn->_base.port,
01010                               digest,
01011                               SESSION_GROUP_DIRCONN, iso_flags,
01012                               use_begindir, conn->dirconn_direct);
01013     if (!linked_conn) {
01014       log_warn(LD_NET,"Making tunnel to dirserver failed.");
01015       connection_mark_for_close(TO_CONN(conn));
01016       return;
01017     }
01018 
01019     if (connection_add(TO_CONN(conn)) < 0) {
01020       log_warn(LD_NET,"Unable to add connection for link to dirserver.");
01021       connection_mark_for_close(TO_CONN(conn));
01022       return;
01023     }
01024     conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
01025     /* queue the command on the outbuf */
01026     directory_send_command(conn, dir_purpose, 0, resource,
01027                            payload, payload_len,
01028                            supports_conditional_consensus,
01029                            if_modified_since);
01030 
01031     connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
01032     IF_HAS_BUFFEREVENT(ENTRY_TO_CONN(linked_conn), {
01033       connection_watch_events(ENTRY_TO_CONN(linked_conn),
01034                               READ_EVENT|WRITE_EVENT);
01035     }) ELSE_IF_NO_BUFFEREVENT
01036       connection_start_reading(ENTRY_TO_CONN(linked_conn));
01037   }
01038 }
01039 
01042 int
01043 connection_dir_is_encrypted(dir_connection_t *conn)
01044 {
01045   /* Right now it's sufficient to see if conn is or has been linked, since
01046    * the only thing it could be linked to is an edge connection on a
01047    * circuit, and the only way it could have been unlinked is at the edge
01048    * connection getting closed.
01049    */
01050   return TO_CONN(conn)->linked;
01051 }
01052 
01057 static int
01058 _compare_strs(const void **a, const void **b)
01059 {
01060   const char *s1 = *a, *s2 = *b;
01061   return strcmp(s1, s2);
01062 }
01063 
01064 #define CONDITIONAL_CONSENSUS_FPR_LEN 3
01065 #if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
01066 #error "conditional consensus fingerprint length is larger than digest length"
01067 #endif
01068 
01077 static char *
01078 directory_get_consensus_url(int supports_conditional_consensus,
01079                             const char *resource)
01080 {
01081   char *url = NULL;
01082   const char *hyphen, *flavor;
01083   if (resource==NULL || strcmp(resource, "ns")==0) {
01084     flavor = ""; /* Request ns consensuses as "", so older servers will work*/
01085     hyphen = "";
01086   } else {
01087     flavor = resource;
01088     hyphen = "-";
01089   }
01090 
01091   if (supports_conditional_consensus) {
01092     char *authority_id_list;
01093     smartlist_t *authority_digests = smartlist_new();
01094 
01095     SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
01096                       trusted_dir_server_t *, ds,
01097       {
01098         char *hex;
01099         if (!(ds->type & V3_DIRINFO))
01100           continue;
01101 
01102         hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
01103         base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
01104                       ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
01105         smartlist_add(authority_digests, hex);
01106       });
01107     smartlist_sort(authority_digests, _compare_strs);
01108     authority_id_list = smartlist_join_strings(authority_digests,
01109                                                "+", 0, NULL);
01110 
01111     tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
01112                  hyphen, flavor, authority_id_list);
01113 
01114     SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
01115     smartlist_free(authority_digests);
01116     tor_free(authority_id_list);
01117   } else {
01118     tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s.z",
01119                  hyphen, flavor);
01120   }
01121   return url;
01122 }
01123 
01127 static void
01128 directory_send_command(dir_connection_t *conn,
01129                        int purpose, int direct, const char *resource,
01130                        const char *payload, size_t payload_len,
01131                        int supports_conditional_consensus,
01132                        time_t if_modified_since)
01133 {
01134   char proxystring[256];
01135   char hoststring[128];
01136   smartlist_t *headers = smartlist_new();
01137   char *url;
01138   char request[8192];
01139   const char *httpcommand = NULL;
01140 
01141   tor_assert(conn);
01142   tor_assert(conn->_base.type == CONN_TYPE_DIR);
01143 
01144   tor_free(conn->requested_resource);
01145   if (resource)
01146     conn->requested_resource = tor_strdup(resource);
01147 
01148   /* come up with a string for which Host: we want */
01149   if (conn->_base.port == 80) {
01150     strlcpy(hoststring, conn->_base.address, sizeof(hoststring));
01151   } else {
01152     tor_snprintf(hoststring, sizeof(hoststring),"%s:%d",
01153                  conn->_base.address, conn->_base.port);
01154   }
01155 
01156   /* Format if-modified-since */
01157   if (if_modified_since) {
01158     char b[RFC1123_TIME_LEN+1];
01159     format_rfc1123_time(b, if_modified_since);
01160     smartlist_add_asprintf(headers, "If-Modified-Since: %s\r\n", b);
01161   }
01162 
01163   /* come up with some proxy lines, if we're using one. */
01164   if (direct && get_options()->HTTPProxy) {
01165     char *base64_authenticator=NULL;
01166     const char *authenticator = get_options()->HTTPProxyAuthenticator;
01167 
01168     tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
01169     if (authenticator) {
01170       base64_authenticator = alloc_http_authenticator(authenticator);
01171       if (!base64_authenticator)
01172         log_warn(LD_BUG, "Encoding http authenticator failed");
01173     }
01174     if (base64_authenticator) {
01175       smartlist_add_asprintf(headers,
01176                    "Proxy-Authorization: Basic %s\r\n",
01177                    base64_authenticator);
01178       tor_free(base64_authenticator);
01179     }
01180   } else {
01181     proxystring[0] = 0;
01182   }
01183 
01184   switch (purpose) {
01185     case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS:
01186       tor_assert(resource);
01187       httpcommand = "GET";
01188       tor_asprintf(&url, "/tor/status/%s", resource);
01189       break;
01190     case DIR_PURPOSE_FETCH_CONSENSUS:
01191       /* resource is optional.  If present, it's a flavor name */
01192       tor_assert(!payload);
01193       httpcommand = "GET";
01194       url = directory_get_consensus_url(supports_conditional_consensus,
01195                                         resource);
01196       log_info(LD_DIR, "Downloading consensus from %s using %s",
01197                hoststring, url);
01198       break;
01199     case DIR_PURPOSE_FETCH_CERTIFICATE:
01200       tor_assert(resource);
01201       tor_assert(!payload);
01202       httpcommand = "GET";
01203       tor_asprintf(&url, "/tor/keys/%s", resource);
01204       break;
01205     case DIR_PURPOSE_FETCH_STATUS_VOTE:
01206       tor_assert(resource);
01207       tor_assert(!payload);
01208       httpcommand = "GET";
01209       tor_asprintf(&url, "/tor/status-vote/next/%s.z", resource);
01210       break;
01211     case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
01212       tor_assert(!resource);
01213       tor_assert(!payload);
01214       httpcommand = "GET";
01215       url = tor_strdup("/tor/status-vote/next/consensus-signatures.z");
01216       break;
01217     case DIR_PURPOSE_FETCH_SERVERDESC:
01218       tor_assert(resource);
01219       httpcommand = "GET";
01220       tor_asprintf(&url, "/tor/server/%s", resource);
01221       break;
01222     case DIR_PURPOSE_FETCH_EXTRAINFO:
01223       tor_assert(resource);
01224       httpcommand = "GET";
01225       tor_asprintf(&url, "/tor/extra/%s", resource);
01226       break;
01227     case DIR_PURPOSE_FETCH_MICRODESC:
01228       tor_assert(resource);
01229       httpcommand = "GET";
01230       tor_asprintf(&url, "/tor/micro/%s", resource);
01231       break;
01232     case DIR_PURPOSE_UPLOAD_DIR: {
01233       const char *why = router_get_descriptor_gen_reason();
01234       tor_assert(!resource);
01235       tor_assert(payload);
01236       httpcommand = "POST";
01237       url = tor_strdup("/tor/");
01238       if (why) {
01239         smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
01240       }
01241       break;
01242     }
01243     case DIR_PURPOSE_UPLOAD_VOTE:
01244       tor_assert(!resource);
01245       tor_assert(payload);
01246       httpcommand = "POST";
01247       url = tor_strdup("/tor/post/vote");
01248       break;
01249     case DIR_PURPOSE_UPLOAD_SIGNATURES:
01250       tor_assert(!resource);
01251       tor_assert(payload);
01252       httpcommand = "POST";
01253       url = tor_strdup("/tor/post/consensus-signature");
01254       break;
01255     case DIR_PURPOSE_FETCH_RENDDESC_V2:
01256       tor_assert(resource);
01257       tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
01258       tor_assert(!payload);
01259       httpcommand = "GET";
01260       tor_asprintf(&url, "/tor/rendezvous2/%s", resource);
01261       break;
01262     case DIR_PURPOSE_UPLOAD_RENDDESC:
01263       tor_assert(!resource);
01264       tor_assert(payload);
01265       httpcommand = "POST";
01266       url = tor_strdup("/tor/rendezvous/publish");
01267       break;
01268     case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
01269       tor_assert(!resource);
01270       tor_assert(payload);
01271       httpcommand = "POST";
01272       url = tor_strdup("/tor/rendezvous2/publish");
01273       break;
01274     default:
01275       tor_assert(0);
01276       return;
01277   }
01278 
01279   if (strlen(proxystring) + strlen(url) >= 4096) {
01280     log_warn(LD_BUG,
01281              "Squid does not like URLs longer than 4095 bytes, and this "
01282              "one is %d bytes long: %s%s",
01283              (int)(strlen(proxystring) + strlen(url)), proxystring, url);
01284   }
01285 
01286   tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
01287   connection_write_to_buf(request, strlen(request), TO_CONN(conn));
01288   connection_write_to_buf(url, strlen(url), TO_CONN(conn));
01289   tor_free(url);
01290 
01291   if (!strcmp(httpcommand, "POST") || payload) {
01292     smartlist_add_asprintf(headers, "Content-Length: %lu\r\n",
01293                  payload ? (unsigned long)payload_len : 0);
01294   }
01295 
01296   {
01297     char *header = smartlist_join_strings(headers, "", 0, NULL);
01298     tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
01299                  hoststring, header);
01300     tor_free(header);
01301   }
01302 
01303   connection_write_to_buf(request, strlen(request), TO_CONN(conn));
01304 
01305   if (payload) {
01306     /* then send the payload afterwards too */
01307     connection_write_to_buf(payload, payload_len, TO_CONN(conn));
01308   }
01309 
01310   SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
01311   smartlist_free(headers);
01312 }
01313 
01323 static int
01324 parse_http_url(const char *headers, char **url)
01325 {
01326   char *s, *start, *tmp;
01327 
01328   s = (char *)eat_whitespace_no_nl(headers);
01329   if (!*s) return -1;
01330   s = (char *)find_whitespace(s); /* get past GET/POST */
01331   if (!*s) return -1;
01332   s = (char *)eat_whitespace_no_nl(s);
01333   if (!*s) return -1;
01334   start = s; /* this is it, assuming it's valid */
01335   s = (char *)find_whitespace(start);
01336   if (!*s) return -1;
01337 
01338   /* tolerate the http[s] proxy style of putting the hostname in the url */
01339   if (s-start >= 4 && !strcmpstart(start,"http")) {
01340     tmp = start + 4;
01341     if (*tmp == 's')
01342       tmp++;
01343     if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
01344       tmp = strchr(tmp+3, '/');
01345       if (tmp && tmp < s) {
01346         log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
01347         start = tmp;
01348       }
01349     }
01350   }
01351 
01352   if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */
01353     *url = tor_malloc(s - start + 5);
01354     strlcpy(*url,"/tor", s-start+5);
01355     strlcat((*url)+4, start, s-start+1);
01356   } else {
01357     *url = tor_strndup(start, s-start);
01358   }
01359   return 0;
01360 }
01361 
01366 static char *
01367 http_get_header(const char *headers, const char *which)
01368 {
01369   const char *cp = headers;
01370   while (cp) {
01371     if (!strcasecmpstart(cp, which)) {
01372       char *eos;
01373       cp += strlen(which);
01374       if ((eos = strchr(cp,'\r')))
01375         return tor_strndup(cp, eos-cp);
01376       else
01377         return tor_strdup(cp);
01378     }
01379     cp = strchr(cp, '\n');
01380     if (cp)
01381       ++cp;
01382   }
01383   return NULL;
01384 }
01385 
01389 static void
01390 http_set_address_origin(const char *headers, connection_t *conn)
01391 {
01392   char *fwd;
01393 
01394   fwd = http_get_header(headers, "Forwarded-For: ");
01395   if (!fwd)
01396     fwd = http_get_header(headers, "X-Forwarded-For: ");
01397   if (fwd) {
01398     struct in_addr in;
01399     if (!tor_inet_aton(fwd, &in) || is_internal_IP(ntohl(in.s_addr), 0)) {
01400       log_debug(LD_DIR, "Ignoring unrecognized or internal IP %s",
01401                 escaped(fwd));
01402       tor_free(fwd);
01403       return;
01404     }
01405     tor_free(conn->address);
01406     conn->address = tor_strdup(fwd);
01407     tor_free(fwd);
01408   }
01409 }
01410 
01427 int
01428 parse_http_response(const char *headers, int *code, time_t *date,
01429                     compress_method_t *compression, char **reason)
01430 {
01431   unsigned n1, n2;
01432   char datestr[RFC1123_TIME_LEN+1];
01433   smartlist_t *parsed_headers;
01434   tor_assert(headers);
01435   tor_assert(code);
01436 
01437   while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
01438 
01439   if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
01440       (n1 != 0 && n1 != 1) ||
01441       (n2 < 100 || n2 >= 600)) {
01442     log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
01443     return -1;
01444   }
01445   *code = n2;
01446 
01447   parsed_headers = smartlist_new();
01448   smartlist_split_string(parsed_headers, headers, "\n",
01449                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
01450   if (reason) {
01451     smartlist_t *status_line_elements = smartlist_new();
01452     tor_assert(smartlist_len(parsed_headers));
01453     smartlist_split_string(status_line_elements,
01454                            smartlist_get(parsed_headers, 0),
01455                            " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
01456     tor_assert(smartlist_len(status_line_elements) <= 3);
01457     if (smartlist_len(status_line_elements) == 3) {
01458       *reason = smartlist_get(status_line_elements, 2);
01459       smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
01460     }
01461     SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
01462     smartlist_free(status_line_elements);
01463   }
01464   if (date) {
01465     *date = 0;
01466     SMARTLIST_FOREACH(parsed_headers, const char *, s,
01467       if (!strcmpstart(s, "Date: ")) {
01468         strlcpy(datestr, s+6, sizeof(datestr));
01469         /* This will do nothing on failure, so we don't need to check
01470            the result.   We shouldn't warn, since there are many other valid
01471            date formats besides the one we use. */
01472         parse_rfc1123_time(datestr, date);
01473         break;
01474       });
01475   }
01476   if (compression) {
01477     const char *enc = NULL;
01478     SMARTLIST_FOREACH(parsed_headers, const char *, s,
01479       if (!strcmpstart(s, "Content-Encoding: ")) {
01480         enc = s+18; break;
01481       });
01482     if (!enc || !strcmp(enc, "identity")) {
01483       *compression = NO_METHOD;
01484     } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
01485       *compression = ZLIB_METHOD;
01486     } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
01487       *compression = GZIP_METHOD;
01488     } else {
01489       log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
01490                escaped(enc));
01491       *compression = UNKNOWN_METHOD;
01492     }
01493   }
01494   SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
01495   smartlist_free(parsed_headers);
01496 
01497   return 0;
01498 }
01499 
01503 static int
01504 body_is_plausible(const char *body, size_t len, int purpose)
01505 {
01506   int i;
01507   if (len == 0)
01508     return 1; /* empty bodies don't need decompression */
01509   if (len < 32)
01510     return 0;
01511   if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
01512     return (!strcmpstart(body,"onion-key"));
01513   }
01514   if (purpose != DIR_PURPOSE_FETCH_RENDDESC) {
01515     if (!strcmpstart(body,"router") ||
01516         !strcmpstart(body,"signed-directory") ||
01517         !strcmpstart(body,"network-status") ||
01518         !strcmpstart(body,"running-routers"))
01519     return 1;
01520     for (i=0;i<32;++i) {
01521       if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
01522         return 0;
01523     }
01524     return 1;
01525   } else {
01526     return 1;
01527   }
01528 }
01529 
01538 static int
01539 load_downloaded_routers(const char *body, smartlist_t *which,
01540                         int descriptor_digests,
01541                         int router_purpose,
01542                         const char *source)
01543 {
01544   char buf[256];
01545   char time_buf[ISO_TIME_LEN+1];
01546   int added = 0;
01547   int general = router_purpose == ROUTER_PURPOSE_GENERAL;
01548   format_iso_time(time_buf, time(NULL));
01549   tor_assert(source);
01550 
01551   if (tor_snprintf(buf, sizeof(buf),
01552                    "@downloaded-at %s\n"
01553                    "@source %s\n"
01554                    "%s%s%s", time_buf, escaped(source),
01555                    !general ? "@purpose " : "",
01556                    !general ? router_purpose_to_string(router_purpose) : "",
01557                    !general ? "\n" : "")<0)
01558     return added;
01559 
01560   added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
01561                                   descriptor_digests, buf);
01562   control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
01563                           count_loading_descriptors_progress());
01564   return added;
01565 }
01566 
01576 static int
01577 connection_dir_client_reached_eof(dir_connection_t *conn)
01578 {
01579   char *body;
01580   char *headers;
01581   char *reason = NULL;
01582   size_t body_len=0, orig_len=0;
01583   int status_code;
01584   time_t date_header=0;
01585   long delta;
01586   compress_method_t compression;
01587   int plausible;
01588   int skewed=0;
01589   int allow_partial = (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
01590                        conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
01591                        conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);
01592   int was_compressed=0;
01593   time_t now = time(NULL);
01594 
01595   switch (connection_fetch_from_buf_http(TO_CONN(conn),
01596                               &headers, MAX_HEADERS_SIZE,
01597                               &body, &body_len, MAX_DIR_DL_SIZE,
01598                               allow_partial)) {
01599     case -1: /* overflow */
01600       log_warn(LD_PROTOCOL,
01601                "'fetch' response too large (server '%s:%d'). Closing.",
01602                conn->_base.address, conn->_base.port);
01603       return -1;
01604     case 0:
01605       log_info(LD_HTTP,
01606                "'fetch' response not all here, but we're at eof. Closing.");
01607       return -1;
01608     /* case 1, fall through */
01609   }
01610   orig_len = body_len;
01611 
01612   if (parse_http_response(headers, &status_code, &date_header,
01613                           &compression, &reason) < 0) {
01614     log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
01615              conn->_base.address, conn->_base.port);
01616     tor_free(body); tor_free(headers);
01617     return -1;
01618   }
01619   if (!reason) reason = tor_strdup("[no reason given]");
01620 
01621   log_debug(LD_DIR,
01622             "Received response from directory server '%s:%d': %d %s "
01623             "(purpose: %d)",
01624             conn->_base.address, conn->_base.port, status_code,
01625             escaped(reason),
01626             conn->_base.purpose);
01627 
01628   /* now check if it's got any hints for us about our IP address. */
01629   if (conn->dirconn_direct) {
01630     char *guess = http_get_header(headers, X_ADDRESS_HEADER);
01631     if (guess) {
01632       router_new_address_suggestion(guess, conn);
01633       tor_free(guess);
01634     }
01635   }
01636 
01637   if (date_header > 0) {
01638     /* The date header was written very soon after we sent our request,
01639      * so compute the skew as the difference between sending the request
01640      * and the date header.  (We used to check now-date_header, but that's
01641      * inaccurate if we spend a lot of time downloading.)
01642      */
01643     delta = conn->_base.timestamp_lastwritten - date_header;
01644     if (labs(delta)>ALLOW_DIRECTORY_TIME_SKEW) {
01645       char dbuf[64];
01646       int trusted = router_digest_is_trusted_dir(conn->identity_digest);
01647       format_time_interval(dbuf, sizeof(dbuf), delta);
01648       log_fn(trusted ? LOG_WARN : LOG_INFO,
01649              LD_HTTP,
01650              "Received directory with skewed time (server '%s:%d'): "
01651              "It seems that our clock is %s by %s, or that theirs is %s. "
01652              "Tor requires an accurate clock to work: please check your time, "
01653              "timezone, and date settings.",
01654              conn->_base.address, conn->_base.port,
01655              delta>0 ? "ahead" : "behind", dbuf,
01656              delta>0 ? "behind" : "ahead");
01657       skewed = 1; /* don't check the recommended-versions line */
01658       if (trusted)
01659         control_event_general_status(LOG_WARN,
01660                                  "CLOCK_SKEW SKEW=%ld SOURCE=DIRSERV:%s:%d",
01661                                  delta, conn->_base.address, conn->_base.port);
01662     } else {
01663       log_debug(LD_HTTP, "Time on received directory is within tolerance; "
01664                 "we are %ld seconds skewed.  (That's okay.)", delta);
01665     }
01666   }
01667   (void) skewed; /* skewed isn't used yet. */
01668 
01669   if (status_code == 503) {
01670     routerstatus_t *rs;
01671     trusted_dir_server_t *ds;
01672     const char *id_digest = conn->identity_digest;
01673     log_info(LD_DIR,"Received http status code %d (%s) from server "
01674              "'%s:%d'. I'll try again soon.",
01675              status_code, escaped(reason), conn->_base.address,
01676              conn->_base.port);
01677     if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
01678       rs->last_dir_503_at = now;
01679     if ((ds = router_get_trusteddirserver_by_digest(id_digest)))
01680       ds->fake_status.last_dir_503_at = now;
01681 
01682     tor_free(body); tor_free(headers); tor_free(reason);
01683     return -1;
01684   }
01685 
01686   plausible = body_is_plausible(body, body_len, conn->_base.purpose);
01687   if (compression != NO_METHOD || !plausible) {
01688     char *new_body = NULL;
01689     size_t new_len = 0;
01690     compress_method_t guessed = detect_compression_method(body, body_len);
01691     if (compression == UNKNOWN_METHOD || guessed != compression) {
01692       /* Tell the user if we don't believe what we're told about compression.*/
01693       const char *description1, *description2;
01694       if (compression == ZLIB_METHOD)
01695         description1 = "as deflated";
01696       else if (compression == GZIP_METHOD)
01697         description1 = "as gzipped";
01698       else if (compression == NO_METHOD)
01699         description1 = "as uncompressed";
01700       else
01701         description1 = "with an unknown Content-Encoding";
01702       if (guessed == ZLIB_METHOD)
01703         description2 = "deflated";
01704       else if (guessed == GZIP_METHOD)
01705         description2 = "gzipped";
01706       else if (!plausible)
01707         description2 = "confusing binary junk";
01708       else
01709         description2 = "uncompressed";
01710 
01711       log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, "
01712                "but it seems to be %s.%s",
01713                conn->_base.address, conn->_base.port, description1,
01714                description2,
01715                (compression>0 && guessed>0)?"  Trying both.":"");
01716     }
01717     /* Try declared compression first if we can. */
01718     if (compression == GZIP_METHOD  || compression == ZLIB_METHOD)
01719       tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression,
01720                           !allow_partial, LOG_PROTOCOL_WARN);
01721     /* Okay, if that didn't work, and we think that it was compressed
01722      * differently, try that. */
01723     if (!new_body &&
01724         (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) &&
01725         compression != guessed)
01726       tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed,
01727                           !allow_partial, LOG_PROTOCOL_WARN);
01728     /* If we're pretty sure that we have a compressed directory, and
01729      * we didn't manage to uncompress it, then warn and bail. */
01730     if (!plausible && !new_body) {
01731       log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
01732              "Unable to decompress HTTP body (server '%s:%d').",
01733              conn->_base.address, conn->_base.port);
01734       tor_free(body); tor_free(headers); tor_free(reason);
01735       return -1;
01736     }
01737     if (new_body) {
01738       tor_free(body);
01739       body = new_body;
01740       body_len = new_len;
01741       was_compressed = 1;
01742     }
01743   }
01744 
01745   if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) {
01746     smartlist_t *which = NULL;
01747     v2_networkstatus_source_t source;
01748     char *cp;
01749     log_info(LD_DIR,"Received networkstatus objects (size %d) from server "
01750              "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
01751     if (status_code != 200) {
01752       static ratelim_t warning_limit = RATELIM_INIT(3600);
01753       char *m;
01754       if ((m = rate_limit_log(&warning_limit, now))) {
01755         log_warn(LD_DIR,
01756                  "Received http status code %d (%s) from server "
01757                  "'%s:%d' while fetching \"/tor/status/%s\". "
01758                  "I'll try again soon.%s",
01759                  status_code, escaped(reason), conn->_base.address,
01760                  conn->_base.port, conn->requested_resource, m);
01761         tor_free(m);
01762       }
01763       tor_free(body); tor_free(headers); tor_free(reason);
01764       connection_dir_download_v2_networkstatus_failed(conn, status_code);
01765       return -1;
01766     }
01767     if (conn->requested_resource &&
01768         !strcmpstart(conn->requested_resource,"fp/")) {
01769       source = NS_FROM_DIR_BY_FP;
01770       which = smartlist_new();
01771       dir_split_resource_into_fingerprints(conn->requested_resource+3,
01772                                            which, NULL, 0);
01773     } else if (conn->requested_resource &&
01774                !strcmpstart(conn->requested_resource, "all")) {
01775       source = NS_FROM_DIR_ALL;
01776       which = smartlist_new();
01777       SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
01778                         trusted_dir_server_t *, ds,
01779         {
01780           char *hex = tor_malloc(HEX_DIGEST_LEN+1);
01781           base16_encode(hex, HEX_DIGEST_LEN+1, ds->digest, DIGEST_LEN);
01782           smartlist_add(which, hex);
01783         });
01784     } else {
01785       /* XXXX Can we even end up here? -- weasel*/
01786       source = NS_FROM_DIR_BY_FP;
01787       log_warn(LD_BUG, "We received a networkstatus but we didn't ask "
01788                        "for it by fp, nor did we ask for all.");
01789     }
01790     cp = body;
01791     while (*cp) {
01792       char *next = strstr(cp, "\nnetwork-status-version");
01793       if (next)
01794         next[1] = '\0';
01795       /* learn from it, and then remove it from 'which' */
01796       if (router_set_networkstatus_v2(cp, now, source, which)<0)
01797         break;
01798       if (next) {
01799         next[1] = 'n';
01800         cp = next+1;
01801       } else
01802         break;
01803     }
01804     /* launches router downloads as needed */
01805     routers_update_all_from_networkstatus(now, 2);
01806     directory_info_has_arrived(now, 0);
01807     if (which) {
01808       if (smartlist_len(which)) {
01809         dir_networkstatus_download_failed(which, status_code);
01810       }
01811       SMARTLIST_FOREACH(which, char *, s, tor_free(s));
01812       smartlist_free(which);
01813     }
01814   }
01815 
01816   if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
01817     int r;
01818     const char *flavname = conn->requested_resource;
01819     if (status_code != 200) {
01820       int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
01821       log(severity, LD_DIR,
01822           "Received http status code %d (%s) from server "
01823           "'%s:%d' while fetching consensus directory.",
01824            status_code, escaped(reason), conn->_base.address,
01825            conn->_base.port);
01826       tor_free(body); tor_free(headers); tor_free(reason);
01827       networkstatus_consensus_download_failed(status_code, flavname);
01828       return -1;
01829     }
01830     log_info(LD_DIR,"Received consensus directory (size %d) from server "
01831              "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
01832     if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) {
01833       log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
01834              "Unable to load %s consensus directory downloaded from "
01835              "server '%s:%d'. I'll try again soon.",
01836              flavname, conn->_base.address, conn->_base.port);
01837       tor_free(body); tor_free(headers); tor_free(reason);
01838       networkstatus_consensus_download_failed(0, flavname);
01839       return -1;
01840     }
01841     /* launches router downloads as needed */
01842     routers_update_all_from_networkstatus(now, 3);
01843     update_microdescs_from_networkstatus(now);
01844     update_microdesc_downloads(now);
01845     directory_info_has_arrived(now, 0);
01846     log_info(LD_DIR, "Successfully loaded consensus.");
01847   }
01848 
01849   if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
01850     if (status_code != 200) {
01851       log_warn(LD_DIR,
01852           "Received http status code %d (%s) from server "
01853           "'%s:%d' while fetching \"/tor/keys/%s\".",
01854            status_code, escaped(reason), conn->_base.address,
01855            conn->_base.port, conn->requested_resource);
01856       connection_dir_download_cert_failed(conn, status_code);
01857       tor_free(body); tor_free(headers); tor_free(reason);
01858       return -1;
01859     }
01860     log_info(LD_DIR,"Received authority certificates (size %d) from server "
01861              "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
01862     if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) {
01863       log_warn(LD_DIR, "Unable to parse fetched certificates");
01864       /* if we fetched more than one and only some failed, the successful
01865        * ones got flushed to disk so it's safe to call this on them */
01866       connection_dir_download_cert_failed(conn, status_code);
01867     } else {
01868       directory_info_has_arrived(now, 0);
01869       log_info(LD_DIR, "Successfully loaded certificates from fetch.");
01870     }
01871   }
01872   if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
01873     const char *msg;
01874     int st;
01875     log_info(LD_DIR,"Got votes (size %d) from server %s:%d",
01876              (int)body_len, conn->_base.address, conn->_base.port);
01877     if (status_code != 200) {
01878       log_warn(LD_DIR,
01879              "Received http status code %d (%s) from server "
01880              "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
01881              status_code, escaped(reason), conn->_base.address,
01882              conn->_base.port, conn->requested_resource);
01883       tor_free(body); tor_free(headers); tor_free(reason);
01884       return -1;
01885     }
01886     dirvote_add_vote(body, &msg, &st);
01887     if (st > 299) {
01888       log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
01889     } else {
01890       log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
01891     }
01892   }
01893   if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
01894     const char *msg = NULL;
01895     log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d",
01896              (int)body_len, conn->_base.address, conn->_base.port);
01897     if (status_code != 200) {
01898       log_warn(LD_DIR,
01899         "Received http status code %d (%s) from server '%s:%d' while fetching "
01900         "\"/tor/status-vote/next/consensus-signatures.z\".",
01901              status_code, escaped(reason), conn->_base.address,
01902              conn->_base.port);
01903       tor_free(body); tor_free(headers); tor_free(reason);
01904       return -1;
01905     }
01906     if (dirvote_add_signatures(body, conn->_base.address, &msg)<0) {
01907       log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
01908                conn->_base.address, conn->_base.port, msg?msg:"???");
01909     }
01910   }
01911 
01912   if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
01913       conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
01914     int was_ei = conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
01915     smartlist_t *which = NULL;
01916     int n_asked_for = 0;
01917     int descriptor_digests = conn->requested_resource &&
01918                              !strcmpstart(conn->requested_resource,"d/");
01919     log_info(LD_DIR,"Received %s (size %d) from server '%s:%d'",
01920              was_ei ? "extra server info" : "server info",
01921              (int)body_len, conn->_base.address, conn->_base.port);
01922     if (conn->requested_resource &&
01923         (!strcmpstart(conn->requested_resource,"d/") ||
01924          !strcmpstart(conn->requested_resource,"fp/"))) {
01925       which = smartlist_new();
01926       dir_split_resource_into_fingerprints(conn->requested_resource +
01927                                              (descriptor_digests ? 2 : 3),
01928                                            which, NULL, 0);
01929       n_asked_for = smartlist_len(which);
01930     }
01931     if (status_code != 200) {
01932       int dir_okay = status_code == 404 ||
01933         (status_code == 400 && !strcmp(reason, "Servers unavailable."));
01934       /* 404 means that it didn't have them; no big deal.
01935        * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
01936       log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
01937              "Received http status code %d (%s) from server '%s:%d' "
01938              "while fetching \"/tor/server/%s\". I'll try again soon.",
01939              status_code, escaped(reason), conn->_base.address,
01940              conn->_base.port, conn->requested_resource);
01941       if (!which) {
01942         connection_dir_download_routerdesc_failed(conn);
01943       } else {
01944         dir_routerdesc_download_failed(which, status_code,
01945                                        conn->router_purpose,
01946                                        was_ei, descriptor_digests);
01947         SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
01948         smartlist_free(which);
01949       }
01950       tor_free(body); tor_free(headers); tor_free(reason);
01951       return dir_okay ? 0 : -1;
01952     }
01953     /* Learn the routers, assuming we requested by fingerprint or "all"
01954      * or "authority".
01955      *
01956      * We use "authority" to fetch our own descriptor for
01957      * testing, and to fetch bridge descriptors for bootstrapping. Ignore
01958      * the output of "authority" requests unless we are using bridges,
01959      * since otherwise they'll be the response from reachability tests,
01960      * and we don't really want to add that to our routerlist. */
01961     if (which || (conn->requested_resource &&
01962                   (!strcmpstart(conn->requested_resource, "all") ||
01963                    (!strcmpstart(conn->requested_resource, "authority") &&
01964                     get_options()->UseBridges)))) {
01965       /* as we learn from them, we remove them from 'which' */
01966       if (was_ei) {
01967         router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
01968                                           descriptor_digests);
01969       } else {
01970         //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
01971         //                       descriptor_digests, conn->router_purpose);
01972         if (load_downloaded_routers(body, which, descriptor_digests,
01973                                 conn->router_purpose,
01974                                 conn->_base.address))
01975           directory_info_has_arrived(now, 0);
01976       }
01977     }
01978     if (which) { /* mark remaining ones as failed */
01979       log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
01980                n_asked_for-smartlist_len(which), n_asked_for,
01981                was_ei ? "extra-info documents" : "router descriptors",
01982                conn->_base.address, (int)conn->_base.port);
01983       if (smartlist_len(which)) {
01984         dir_routerdesc_download_failed(which, status_code,
01985                                        conn->router_purpose,
01986                                        was_ei, descriptor_digests);
01987       }
01988       SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
01989       smartlist_free(which);
01990     }
01991     if (directory_conn_is_self_reachability_test(conn))
01992       router_dirport_found_reachable();
01993   }
01994   if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
01995     smartlist_t *which = NULL;
01996     log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
01997              "size %d) from server '%s:%d'",
01998              status_code, (int)body_len, conn->_base.address,
01999              conn->_base.port);
02000     tor_assert(conn->requested_resource &&
02001                !strcmpstart(conn->requested_resource, "d/"));
02002     which = smartlist_new();
02003     dir_split_resource_into_fingerprints(conn->requested_resource+2,
02004                                          which, NULL,
02005                                          DSR_DIGEST256|DSR_BASE64);
02006     if (status_code != 200) {
02007       log_info(LD_DIR, "Received status code %d (%s) from server "
02008                "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
02009                "soon.",
02010                status_code, escaped(reason), conn->_base.address,
02011                (int)conn->_base.port, conn->requested_resource);
02012       dir_microdesc_download_failed(which, status_code);
02013       SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
02014       smartlist_free(which);
02015       tor_free(body); tor_free(headers); tor_free(reason);
02016       return 0;
02017     } else {
02018       smartlist_t *mds;
02019       mds = microdescs_add_to_cache(get_microdesc_cache(),
02020                                     body, body+body_len, SAVED_NOWHERE, 0,
02021                                     now, which);
02022       if (smartlist_len(which)) {
02023         /* Mark remaining ones as failed. */
02024         dir_microdesc_download_failed(which, status_code);
02025       }
02026       SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
02027       smartlist_free(which);
02028       smartlist_free(mds);
02029     }
02030   }
02031 
02032   if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_DIR) {
02033     switch (status_code) {
02034       case 200: {
02035           trusted_dir_server_t *ds =
02036             router_get_trusteddirserver_by_digest(conn->identity_digest);
02037           char *rejected_hdr = http_get_header(headers,
02038                                                "X-Descriptor-Not-New: ");
02039           if (rejected_hdr) {
02040             if (!strcmp(rejected_hdr, "Yes")) {
02041               log_info(LD_GENERAL,
02042                        "Authority '%s' declined our descriptor (not new)",
02043                        ds->nickname);
02044               /* XXXX use this information; be sure to upload next one
02045                * sooner. -NM */
02046               /* XXXX023 On further thought, the task above implies that we're
02047                * basing our regenerate-descriptor time on when we uploaded the
02048                * last descriptor, not on the published time of the last
02049                * descriptor.  If those are different, that's a bad thing to
02050                * do. -NM */
02051             }
02052             tor_free(rejected_hdr);
02053           }
02054           log_info(LD_GENERAL,"eof (status 200) after uploading server "
02055                    "descriptor: finished.");
02056           control_event_server_status(
02057                       LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
02058                       conn->_base.address, conn->_base.port);
02059 
02060           ds->has_accepted_serverdesc = 1;
02061           if (directories_have_accepted_server_descriptor())
02062             control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
02063         }
02064         break;
02065       case 400:
02066         log_warn(LD_GENERAL,"http status 400 (%s) response from "
02067                  "dirserver '%s:%d'. Please correct.",
02068                  escaped(reason), conn->_base.address, conn->_base.port);
02069         control_event_server_status(LOG_WARN,
02070                       "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
02071                       conn->_base.address, conn->_base.port, escaped(reason));
02072         break;
02073       default:
02074         log_warn(LD_GENERAL,
02075              "http status %d (%s) reason unexpected while uploading "
02076              "descriptor to server '%s:%d').",
02077              status_code, escaped(reason), conn->_base.address,
02078              conn->_base.port);
02079         break;
02080     }
02081     /* return 0 in all cases, since we don't want to mark any
02082      * dirservers down just because they don't like us. */
02083   }
02084 
02085   if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_VOTE) {
02086     switch (status_code) {
02087       case 200: {
02088         log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
02089                    conn->_base.address, conn->_base.port);
02090         }
02091         break;
02092       case 400:
02093         log_warn(LD_DIR,"http status 400 (%s) response after uploading "
02094                  "vote to dirserver '%s:%d'. Please correct.",
02095                  escaped(reason), conn->_base.address, conn->_base.port);
02096         break;
02097       default:
02098         log_warn(LD_GENERAL,
02099              "http status %d (%s) reason unexpected while uploading "
02100              "vote to server '%s:%d').",
02101              status_code, escaped(reason), conn->_base.address,
02102              conn->_base.port);
02103         break;
02104     }
02105     /* return 0 in all cases, since we don't want to mark any
02106      * dirservers down just because they don't like us. */
02107   }
02108 
02109   if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES) {
02110     switch (status_code) {
02111       case 200: {
02112         log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
02113                    conn->_base.address, conn->_base.port);
02114         }
02115         break;
02116       case 400:
02117         log_warn(LD_DIR,"http status 400 (%s) response after uploading "
02118                  "signatures to dirserver '%s:%d'. Please correct.",
02119                  escaped(reason), conn->_base.address, conn->_base.port);
02120         break;
02121       default:
02122         log_warn(LD_GENERAL,
02123              "http status %d (%s) reason unexpected while uploading "
02124              "signatures to server '%s:%d').",
02125              status_code, escaped(reason), conn->_base.address,
02126              conn->_base.port);
02127         break;
02128     }
02129     /* return 0 in all cases, since we don't want to mark any
02130      * dirservers down just because they don't like us. */
02131   }
02132 
02133   if (conn->_base.purpose == DIR_PURPOSE_FETCH_RENDDESC) {
02134     tor_assert(conn->rend_data);
02135     log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d "
02136              "(%s))",
02137              (int)body_len, status_code, escaped(reason));
02138     switch (status_code) {
02139       case 200:
02140         if (rend_cache_store(body, body_len, 0,
02141                              conn->rend_data->onion_address) < -1) {
02142           log_warn(LD_REND,"Failed to parse rendezvous descriptor.");
02143           /* Any pending rendezvous attempts will notice when
02144            * connection_about_to_close_connection()
02145            * cleans this dir conn up. */
02146           /* We could retry. But since v0 descriptors are going out of
02147            * style, it isn't worth the hassle. We'll do better in v2. */
02148         } else {
02149           /* Success, or at least there's a v2 descriptor already
02150            * present. Notify pending connections about this. */
02151           conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC;
02152           rend_client_desc_trynow(conn->rend_data->onion_address);
02153         }
02154         break;
02155       case 404:
02156         /* Not there. Pending connections will be notified when
02157          * connection_about_to_close_connection() cleans this conn up. */
02158         break;
02159       case 400:
02160         log_warn(LD_REND,
02161                  "http status 400 (%s). Dirserver didn't like our "
02162                  "rendezvous query?", escaped(reason));
02163         break;
02164       default:
02165         log_warn(LD_REND,"http status %d (%s) response unexpected while "
02166                  "fetching hidden service descriptor (server '%s:%d').",
02167                  status_code, escaped(reason), conn->_base.address,
02168                  conn->_base.port);
02169         break;
02170     }
02171   }
02172 
02173   if (conn->_base.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) {
02174     tor_assert(conn->rend_data);
02175     log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d "
02176              "(%s))",
02177              (int)body_len, status_code, escaped(reason));
02178     switch (status_code) {
02179       case 200:
02180         switch (rend_cache_store_v2_desc_as_client(body, conn->rend_data)) {
02181           case -2:
02182             log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
02183                      "Retrying at another directory.");
02184             /* We'll retry when connection_about_to_close_connection()
02185              * cleans this dir conn up. */
02186             break;
02187           case -1:
02188             /* We already have a v0 descriptor here. Ignoring this one
02189              * and _not_ performing another request. */
02190             log_info(LD_REND, "Successfully fetched v2 rendezvous "
02191                      "descriptor, but we already have a v0 descriptor.");
02192             conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC;
02193             break;
02194           default:
02195             /* success. notify pending connections about this. */
02196             log_info(LD_REND, "Successfully fetched v2 rendezvous "
02197                      "descriptor.");
02198             conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC;
02199             rend_client_desc_trynow(conn->rend_data->onion_address);
02200             break;
02201         }
02202         break;
02203       case 404:
02204         /* Not there. We'll retry when
02205          * connection_about_to_close_connection() cleans this conn up. */
02206         log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
02207                          "Retrying at another directory.");
02208         break;
02209       case 400:
02210         log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
02211                  "http status 400 (%s). Dirserver didn't like our "
02212                  "v2 rendezvous query? Retrying at another directory.",
02213                  escaped(reason));
02214         break;
02215       default:
02216         log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
02217                  "http status %d (%s) response unexpected while "
02218                  "fetching v2 hidden service descriptor (server '%s:%d'). "
02219                  "Retrying at another directory.",
02220                  status_code, escaped(reason), conn->_base.address,
02221                  conn->_base.port);
02222         break;
02223     }
02224   }
02225 
02226   if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_RENDDESC ||
02227       conn->_base.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) {
02228     log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
02229              "(%s))",
02230              status_code, escaped(reason));
02231     switch (status_code) {
02232       case 200:
02233         log_info(LD_REND,
02234                  "Uploading rendezvous descriptor: finished with status "
02235                  "200 (%s)", escaped(reason));
02236         break;
02237       case 400:
02238         log_warn(LD_REND,"http status 400 (%s) response from dirserver "
02239                  "'%s:%d'. Malformed rendezvous descriptor?",
02240                  escaped(reason), conn->_base.address, conn->_base.port);
02241         break;
02242       default:
02243         log_warn(LD_REND,"http status %d (%s) response unexpected (server "
02244                  "'%s:%d').",
02245                  status_code, escaped(reason), conn->_base.address,
02246                  conn->_base.port);
02247         break;
02248     }
02249   }
02250   note_client_request(conn->_base.purpose, was_compressed, orig_len);
02251   tor_free(body); tor_free(headers); tor_free(reason);
02252   return 0;
02253 }
02254 
02256 int
02257 connection_dir_reached_eof(dir_connection_t *conn)
02258 {
02259   int retval;
02260   if (conn->_base.state != DIR_CONN_STATE_CLIENT_READING) {
02261     log_info(LD_HTTP,"conn reached eof, not reading. [state=%d] Closing.",
02262              conn->_base.state);
02263     connection_close_immediate(TO_CONN(conn)); /* error: give up on flushing */
02264     connection_mark_for_close(TO_CONN(conn));
02265     return -1;
02266   }
02267 
02268   retval = connection_dir_client_reached_eof(conn);
02269   if (retval == 0) /* success */
02270     conn->_base.state = DIR_CONN_STATE_CLIENT_FINISHED;
02271   connection_mark_for_close(TO_CONN(conn));
02272   return retval;
02273 }
02274 
02279 #define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
02280 
02284 int
02285 connection_dir_process_inbuf(dir_connection_t *conn)
02286 {
02287   tor_assert(conn);
02288   tor_assert(conn->_base.type == CONN_TYPE_DIR);
02289 
02290   /* Directory clients write, then read data until they receive EOF;
02291    * directory servers read data until they get an HTTP command, then
02292    * write their response (when it's finished flushing, they mark for
02293    * close).
02294    */
02295 
02296   /* If we're on the dirserver side, look for a command. */
02297   if (conn->_base.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
02298     if (directory_handle_command(conn) < 0) {
02299       connection_mark_for_close(TO_CONN(conn));
02300       return -1;
02301     }
02302     return 0;
02303   }
02304 
02305   if (connection_get_inbuf_len(TO_CONN(conn)) > MAX_DIRECTORY_OBJECT_SIZE) {
02306     log_warn(LD_HTTP, "Too much data received from directory connection: "
02307              "denial of service attempt, or you need to upgrade?");
02308     connection_mark_for_close(TO_CONN(conn));
02309     return -1;
02310   }
02311 
02312   if (!conn->_base.inbuf_reached_eof)
02313     log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
02314   return 0;
02315 }
02316 
02319 void
02320 connection_dir_about_to_close(dir_connection_t *dir_conn)
02321 {
02322   connection_t *conn = TO_CONN(dir_conn);
02323 
02324   if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
02325     /* It's a directory connection and connecting or fetching
02326      * failed: forget about this router, and maybe try again. */
02327     connection_dir_request_failed(dir_conn);
02328   }
02329   /* If we were trying to fetch a v2 rend desc and did not succeed,
02330    * retry as needed. (If a fetch is successful, the connection state
02331    * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that
02332    * refetching is unnecessary.) */
02333   if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
02334       dir_conn->rend_data &&
02335       strlen(dir_conn->rend_data->onion_address) == REND_SERVICE_ID_LEN_BASE32)
02336     rend_client_refetch_v2_renddesc(dir_conn->rend_data);
02337 }
02338 
02342 static void
02343 write_http_status_line(dir_connection_t *conn, int status,
02344                        const char *reason_phrase)
02345 {
02346   char buf[256];
02347   if (tor_snprintf(buf, sizeof(buf), "HTTP/1.0 %d %s\r\n\r\n",
02348       status, reason_phrase ? reason_phrase : "OK") < 0) {
02349     log_warn(LD_BUG,"status line too long.");
02350     return;
02351   }
02352   connection_write_to_buf(buf, strlen(buf), TO_CONN(conn));
02353 }
02354 
02362 static void
02363 write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
02364                            const char *type, const char *encoding,
02365                            const char *extra_headers,
02366                            long cache_lifetime)
02367 {
02368   char date[RFC1123_TIME_LEN+1];
02369   char tmp[1024];
02370   char *cp;
02371   time_t now = time(NULL);
02372 
02373   tor_assert(conn);
02374 
02375   format_rfc1123_time(date, now);
02376   cp = tmp;
02377   tor_snprintf(cp, sizeof(tmp),
02378                "HTTP/1.0 200 OK\r\nDate: %s\r\n",
02379                date);
02380   cp += strlen(tmp);
02381   if (type) {
02382     tor_snprintf(cp, sizeof(tmp)-(cp-tmp), "Content-Type: %s\r\n", type);
02383     cp += strlen(cp);
02384   }
02385   if (!is_local_addr(&conn->_base.addr)) {
02386     /* Don't report the source address for a nearby/private connection.
02387      * Otherwise we tend to mis-report in cases where incoming ports are
02388      * being forwarded to a Tor server running behind the firewall. */
02389     tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
02390                  X_ADDRESS_HEADER "%s\r\n", conn->_base.address);
02391     cp += strlen(cp);
02392   }
02393   if (encoding) {
02394     tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
02395                  "Content-Encoding: %s\r\n", encoding);
02396     cp += strlen(cp);
02397   }
02398   if (length >= 0) {
02399     tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
02400                  "Content-Length: %ld\r\n", (long)length);
02401     cp += strlen(cp);
02402   }
02403   if (cache_lifetime > 0) {
02404     char expbuf[RFC1123_TIME_LEN+1];
02405     format_rfc1123_time(expbuf, now + cache_lifetime);
02406     /* We could say 'Cache-control: max-age=%d' here if we start doing
02407      * http/1.1 */
02408     tor_snprintf(cp, sizeof(tmp)-(cp-tmp),
02409                  "Expires: %s\r\n", expbuf);
02410     cp += strlen(cp);
02411   } else if (cache_lifetime == 0) {
02412     /* We could say 'Cache-control: no-cache' here if we start doing
02413      * http/1.1 */
02414     strlcpy(cp, "Pragma: no-cache\r\n", sizeof(tmp)-(cp-tmp));
02415     cp += strlen(cp);
02416   }
02417   if (extra_headers) {
02418     strlcpy(cp, extra_headers, sizeof(tmp)-(cp-tmp));
02419     cp += strlen(cp);
02420   }
02421   if (sizeof(tmp)-(cp-tmp) > 3)
02422     memcpy(cp, "\r\n", 3);
02423   else
02424     tor_assert(0);
02425   connection_write_to_buf(tmp, strlen(tmp), TO_CONN(conn));
02426 }
02427 
02430 static void
02431 write_http_response_header(dir_connection_t *conn, ssize_t length,
02432                            int compressed, long cache_lifetime)
02433 {
02434   write_http_response_header_impl(conn, length,
02435                           compressed?"application/octet-stream":"text/plain",
02436                           compressed?"deflate":"identity",
02437                              NULL,
02438                              cache_lifetime);
02439 }
02440 
02441 #if defined(INSTRUMENT_DOWNLOADS) || defined(RUNNING_DOXYGEN)
02442 /* DOCDOC */
02443 typedef struct request_t {
02444   uint64_t bytes; 
02445   uint64_t count; 
02446 } request_t;
02447 
02450 static strmap_t *request_map = NULL;
02451 
02456 static void
02457 note_client_request(int purpose, int compressed, size_t bytes)
02458 {
02459   char *key;
02460   const char *kind = NULL;
02461   switch (purpose) {
02462     case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: kind = "dl/status"; break;
02463     case DIR_PURPOSE_FETCH_CONSENSUS:     kind = "dl/consensus"; break;
02464     case DIR_PURPOSE_FETCH_CERTIFICATE:   kind = "dl/cert"; break;
02465     case DIR_PURPOSE_FETCH_STATUS_VOTE:   kind = "dl/vote"; break;
02466     case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: kind = "dl/detached_sig";
02467          break;
02468     case DIR_PURPOSE_FETCH_SERVERDESC:    kind = "dl/server"; break;
02469     case DIR_PURPOSE_FETCH_EXTRAINFO:     kind = "dl/extra"; break;
02470     case DIR_PURPOSE_UPLOAD_DIR:          kind = "dl/ul-dir"; break;
02471     case DIR_PURPOSE_UPLOAD_VOTE:         kind = "dl/ul-vote"; break;
02472     case DIR_PURPOSE_UPLOAD_SIGNATURES:   kind = "dl/ul-sig"; break;
02473     case DIR_PURPOSE_FETCH_RENDDESC:      kind = "dl/rend"; break;
02474     case DIR_PURPOSE_FETCH_RENDDESC_V2:   kind = "dl/rend2"; break;
02475     case DIR_PURPOSE_UPLOAD_RENDDESC:     kind = "dl/ul-rend"; break;
02476     case DIR_PURPOSE_UPLOAD_RENDDESC_V2:  kind = "dl/ul-rend2"; break;
02477   }
02478   if (kind) {
02479     tor_asprintf(&key, "%s%s", kind, compressed?".z":"");
02480   } else {
02481     tor_asprintf(&key, "unknown purpose (%d)%s",
02482                  purpose, compressed?".z":"");
02483   }
02484   note_request(key, bytes);
02485   tor_free(key);
02486 }
02487 
02489 static void
02490 ensure_request_map_initialized(void)
02491 {
02492   if (!request_map)
02493     request_map = strmap_new();
02494 }
02495 
02499 void
02500 note_request(const char *key, size_t bytes)
02501 {
02502   request_t *r;
02503   ensure_request_map_initialized();
02504 
02505   r = strmap_get(request_map, key);
02506   if (!r) {
02507     r = tor_malloc_zero(sizeof(request_t));
02508     strmap_set(request_map, key, r);
02509   }
02510   r->bytes += bytes;
02511   r->count++;
02512 }
02513 
02516 char *
02517 directory_dump_request_log(void)
02518 {
02519   smartlist_t *lines;
02520   char *result;
02521   strmap_iter_t *iter;
02522 
02523   ensure_request_map_initialized();
02524 
02525   lines = smartlist_new();
02526 
02527   for (iter = strmap_iter_init(request_map);
02528        !strmap_iter_done(iter);
02529        iter = strmap_iter_next(request_map, iter)) {
02530     const char *key;
02531     void *val;
02532     request_t *r;
02533     strmap_iter_get(iter, &key, &val);
02534     r = val;
02535     smartlist_add_asprintf(lines, "%s  "U64_FORMAT"  "U64_FORMAT"\n",
02536                  key, U64_PRINTF_ARG(r->bytes), U64_PRINTF_ARG(r->count));
02537   }
02538   smartlist_sort_strings(lines);
02539   result = smartlist_join_strings(lines, "", 0, NULL);
02540   SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
02541   smartlist_free(lines);
02542   return result;
02543 }
02544 #else
02545 static void
02546 note_client_request(int purpose, int compressed, size_t bytes)
02547 {
02548   (void)purpose;
02549   (void)compressed;
02550   (void)bytes;
02551 }
02552 
02553 void
02554 note_request(const char *key, size_t bytes)
02555 {
02556   (void)key;
02557   (void)bytes;
02558 }
02559 
02560 char *
02561 directory_dump_request_log(void)
02562 {
02563   return tor_strdup("Not supported.");
02564 }
02565 #endif
02566 
02581 int
02582 client_likes_consensus(networkstatus_t *v, const char *want_url)
02583 {
02584   smartlist_t *want_authorities = smartlist_new();
02585   int need_at_least;
02586   int have = 0;
02587 
02588   dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
02589   need_at_least = smartlist_len(want_authorities)/2+1;
02590   SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) {
02591     char want_digest[DIGEST_LEN];
02592     size_t want_len = strlen(d)/2;
02593     if (want_len > DIGEST_LEN)
02594       want_len = DIGEST_LEN;
02595 
02596     if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) {
02597       log_fn(LOG_PROTOCOL_WARN, LD_DIR,
02598              "Failed to decode requested authority digest %s.", d);
02599       continue;
02600     };
02601 
02602     SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) {
02603       if (smartlist_len(vi->sigs) &&
02604           tor_memeq(vi->identity_digest, want_digest, want_len)) {
02605         have++;
02606         break;
02607       };
02608     } SMARTLIST_FOREACH_END(vi);
02609 
02610     /* early exit, if we already have enough */
02611     if (have >= need_at_least)
02612       break;
02613   } SMARTLIST_FOREACH_END(d);
02614 
02615   SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
02616   smartlist_free(want_authorities);
02617   return (have >= need_at_least);
02618 }
02619 
02625 static int
02626 directory_handle_command_get(dir_connection_t *conn, const char *headers,
02627                              const char *req_body, size_t req_body_len)
02628 {
02629   size_t dlen;
02630   char *url, *url_mem, *header;
02631   const or_options_t *options = get_options();
02632   time_t if_modified_since = 0;
02633   int compressed;
02634   size_t url_len;
02635 
02636   /* We ignore the body of a GET request. */
02637   (void)req_body;
02638   (void)req_body_len;
02639 
02640   log_debug(LD_DIRSERV,"Received GET command.");
02641 
02642   conn->_base.state = DIR_CONN_STATE_SERVER_WRITING;
02643 
02644   if (parse_http_url(headers, &url) < 0) {
02645     write_http_status_line(conn, 400, "Bad request");
02646     return 0;
02647   }
02648   if ((header = http_get_header(headers, "If-Modified-Since: "))) {
02649     struct tm tm;
02650     if (parse_http_time(header, &tm) == 0) {
02651       if_modified_since = tor_timegm(&tm);
02652     }
02653     /* The correct behavior on a malformed If-Modified-Since header is to
02654      * act as if no If-Modified-Since header had been given. */
02655     tor_free(header);
02656   }
02657   log_debug(LD_DIRSERV,"rewritten url as '%s'.", url);
02658 
02659   url_mem = url;
02660   url_len = strlen(url);
02661   compressed = url_len > 2 && !strcmp(url+url_len-2, ".z");
02662   if (compressed) {
02663     url[url_len-2] = '\0';
02664     url_len -= 2;
02665   }
02666 
02667   if (!strcmp(url,"/tor/")) {
02668     const char *frontpage = get_dirportfrontpage();
02669 
02670     if (frontpage) {
02671       dlen = strlen(frontpage);
02672       /* Let's return a disclaimer page (users shouldn't use V1 anymore,
02673          and caches don't fetch '/', so this is safe). */
02674 
02675       /* [We don't check for write_bucket_low here, since we want to serve
02676        *  this page no matter what.] */
02677       note_request(url, dlen);
02678       write_http_response_header_impl(conn, dlen, "text/html", "identity",
02679                                       NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
02680       connection_write_to_buf(frontpage, dlen, TO_CONN(conn));
02681       goto done;
02682     }
02683     /* if no disclaimer file, fall through and continue */
02684   }
02685 
02686   if (!strcmp(url,"/tor/") || !strcmp(url,"/tor/dir")) { /* v1 dir fetch */
02687     cached_dir_t *d = dirserv_get_directory();
02688 
02689     if (!d) {
02690       log_info(LD_DIRSERV,"Client asked for the mirrored directory, but we "
02691                "don't have a good one yet. Sending 503 Dir not available.");
02692       write_http_status_line(conn, 503, "Directory unavailable");
02693       goto done;
02694     }
02695     if (d->published < if_modified_since) {
02696       write_http_status_line(conn, 304, "Not modified");
02697       goto done;
02698     }
02699 
02700     dlen = compressed ? d->dir_z_len : d->dir_len;
02701 
02702     if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) {
02703       log_debug(LD_DIRSERV,
02704                "Client asked for the mirrored directory, but we've been "
02705                "writing too many bytes lately. Sending 503 Dir busy.");
02706       write_http_status_line(conn, 503, "Directory busy, try again later");
02707       goto done;
02708     }
02709 
02710     note_request(url, dlen);
02711 
02712     log_debug(LD_DIRSERV,"Dumping %sdirectory to client.",
02713               compressed?"compressed ":"");
02714     write_http_response_header(conn, dlen, compressed,
02715                           FULL_DIR_CACHE_LIFETIME);
02716     conn->cached_dir = d;
02717     conn->cached_dir_offset = 0;
02718     if (!compressed)
02719       conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD);
02720     ++d->refcnt;
02721 
02722     /* Prime the connection with some data. */
02723     conn->dir_spool_src = DIR_SPOOL_CACHED_DIR;
02724     connection_dirserv_flushed_some(conn);
02725     goto done;
02726   }
02727 
02728   if (!strcmp(url,"/tor/running-routers")) { /* running-routers fetch */
02729     cached_dir_t *d = dirserv_get_runningrouters();
02730     if (!d) {
02731       write_http_status_line(conn, 503, "Directory unavailable");
02732       goto done;
02733     }
02734     if (d->published < if_modified_since) {
02735       write_http_status_line(conn, 304, "Not modified");
02736       goto done;
02737     }
02738     dlen = compressed ? d->dir_z_len : d->dir_len;
02739 
02740     if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) {
02741       log_info(LD_DIRSERV,
02742                "Client asked for running-routers, but we've been "
02743                "writing too many bytes lately. Sending 503 Dir busy.");
02744       write_http_status_line(conn, 503, "Directory busy, try again later");
02745       goto done;
02746     }
02747     note_request(url, dlen);
02748     write_http_response_header(conn, dlen, compressed,
02749                  RUNNINGROUTERS_CACHE_LIFETIME);
02750     connection_write_to_buf(compressed ? d->dir_z : d->dir, dlen,
02751                             TO_CONN(conn));
02752     goto done;
02753   }
02754 
02755   if (!strcmpstart(url,"/tor/status/")
02756       || !strcmpstart(url, "/tor/status-vote/current/consensus")) {
02757     /* v2 or v3 network status fetch. */
02758     smartlist_t *dir_fps = smartlist_new();
02759     int is_v3 = !strcmpstart(url, "/tor/status-vote");
02760     geoip_client_action_t act =
02761         is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2;
02762     const char *request_type = NULL;
02763     const char *key = url + strlen("/tor/status/");
02764     long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
02765 
02766     if (!is_v3) {
02767       dirserv_get_networkstatus_v2_fingerprints(dir_fps, key);
02768       if (!strcmpstart(key, "fp/"))
02769         request_type = compressed?"/tor/status/fp.z":"/tor/status/fp";
02770       else if (!strcmpstart(key, "authority"))
02771         request_type = compressed?"/tor/status/authority.z":
02772           "/tor/status/authority";
02773       else if (!strcmpstart(key, "all"))
02774         request_type = compressed?"/tor/status/all.z":"/tor/status/all";
02775       else
02776         request_type = "/tor/status/?";
02777     } else {
02778       networkstatus_t *v;
02779       time_t now = time(NULL);
02780       const char *want_fps = NULL;
02781       char *flavor = NULL;
02782       int flav = FLAV_NS;
02783       #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/"
02784       #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-"
02785       /* figure out the flavor if any, and who we wanted to sign the thing */
02786       if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
02787         const char *f, *cp;
02788         f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
02789         cp = strchr(f, '/');
02790         if (cp) {
02791           want_fps = cp+1;
02792           flavor = tor_strndup(f, cp-f);
02793         } else {
02794           flavor = tor_strdup(f);
02795         }
02796         flav = networkstatus_parse_flavor_name(flavor);
02797         if (flav < 0)
02798           flav = FLAV_NS;
02799       } else {
02800         if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
02801           want_fps = url+strlen(CONSENSUS_URL_PREFIX);
02802       }
02803 
02804       v = networkstatus_get_latest_consensus_by_flavor(flav);
02805 
02806       if (v && want_fps &&
02807           !client_likes_consensus(v, want_fps)) {
02808         write_http_status_line(conn, 404, "Consensus not signed by sufficient "
02809                                           "number of requested authorities");
02810         smartlist_free(dir_fps);
02811         geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS);
02812         tor_free(flavor);
02813         goto done;
02814       }
02815 
02816       {
02817         char *fp = tor_malloc_zero(DIGEST_LEN);
02818         if (flavor)
02819           strlcpy(fp, flavor, DIGEST_LEN);
02820         tor_free(flavor);
02821         smartlist_add(dir_fps, fp);
02822       }
02823       request_type = compressed?"v3.z":"v3";
02824       lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0;
02825     }
02826 
02827     if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */
02828       write_http_status_line(conn, 503, "Network status object unavailable");
02829       smartlist_free(dir_fps);
02830       geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE);
02831       goto done;
02832     }
02833 
02834     if (!dirserv_remove_old_statuses(dir_fps, if_modified_since)) {
02835       write_http_status_line(conn, 404, "Not found");
02836       SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
02837       smartlist_free(dir_fps);
02838       geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND);
02839       goto done;
02840     } else if (!smartlist_len(dir_fps)) {
02841       write_http_status_line(conn, 304, "Not modified");
02842       SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
02843       smartlist_free(dir_fps);
02844       geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED);
02845       goto done;
02846     }
02847 
02848     dlen = dirserv_estimate_data_size(dir_fps, 0, compressed);
02849     if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
02850       log_debug(LD_DIRSERV,
02851                "Client asked for network status lists, but we've been "
02852                "writing too many bytes lately. Sending 503 Dir busy.");
02853       write_http_status_line(conn, 503, "Directory busy, try again later");
02854       SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp));
02855       smartlist_free(dir_fps);
02856       geoip_note_ns_response(act, GEOIP_REJECT_BUSY);
02857       goto done;
02858     }
02859 
02860     {
02861       struct in_addr in;
02862       tor_addr_t addr;
02863       if (tor_inet_aton((TO_CONN(conn))->address, &in)) {
02864         tor_addr_from_ipv4h(&addr, ntohl(in.s_addr));
02865         geoip_note_client_seen(act, &addr, time(NULL));
02866         geoip_note_ns_response(act, GEOIP_SUCCESS);
02867         /* Note that a request for a network status has started, so that we
02868          * can measure the download time later on. */
02869         if (TO_CONN(conn)->dirreq_id)
02870           geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act,
02871                              DIRREQ_TUNNELED);
02872         else
02873           geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act,
02874                              DIRREQ_DIRECT);
02875       }
02876     }
02877 
02878     // note_request(request_type,dlen);
02879     (void) request_type;
02880     write_http_response_header(conn, -1, compressed,
02881                                smartlist_len(dir_fps) == 1 ? lifetime : 0);
02882     conn->fingerprint_stack = dir_fps;
02883     if (! compressed)
02884       conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD);
02885 
02886     /* Prime the connection with some data. */
02887     conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
02888     connection_dirserv_flushed_some(conn);
02889     goto done;
02890   }
02891 
02892   if (!strcmpstart(url,"/tor/status-vote/current/") ||
02893       !strcmpstart(url,"/tor/status-vote/next/")) {
02894     /* XXXX If-modified-since is only implemented for the current
02895      * consensus: that's probably fine, since it's the only vote document
02896      * people fetch much. */
02897     int current;
02898     ssize_t body_len = 0;
02899     ssize_t estimated_len = 0;
02900     smartlist_t *items = smartlist_new();
02901     smartlist_t *dir_items = smartlist_new();
02902     int lifetime = 60; /* XXXX023 should actually use vote intervals. */
02903     url += strlen("/tor/status-vote/");
02904     current = !strcmpstart(url, "current/");
02905     url = strchr(url, '/');
02906     tor_assert(url);
02907     ++url;
02908     if (!strcmp(url, "consensus")) {
02909       const char *item;
02910       tor_assert(!current); /* we handle current consensus specially above,
02911                              * since it wants to be spooled. */
02912       if ((item = dirvote_get_pending_consensus(FLAV_NS)))
02913         smartlist_add(items, (char*)item);
02914     } else if (!current && !strcmp(url, "consensus-signatures")) {
02915       /* XXXX the spec says that we should implement
02916        * current/consensus-signatures too.  It doesn't seem to be needed,
02917        * though. */
02918       const char *item;
02919       if ((item=dirvote_get_pending_detached_signatures()))
02920         smartlist_add(items, (char*)item);
02921     } else if (!strcmp(url, "authority")) {
02922       const cached_dir_t *d;
02923       int flags = DGV_BY_ID |
02924         (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
02925       if ((d=dirvote_get_vote(NULL, flags)))
02926         smartlist_add(dir_items, (cached_dir_t*)d);
02927     } else {
02928       const cached_dir_t *d;
02929       smartlist_t *fps = smartlist_new();
02930       int flags;
02931       if (!strcmpstart(url, "d/")) {
02932         url += 2;
02933         flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS;
02934       } else {
02935         flags = DGV_BY_ID |
02936           (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
02937       }
02938       dir_split_resource_into_fingerprints(url, fps, NULL,
02939                                            DSR_HEX|DSR_SORT_UNIQ);
02940       SMARTLIST_FOREACH(fps, char *, fp, {
02941           if ((d = dirvote_get_vote(fp, flags)))
02942             smartlist_add(dir_items, (cached_dir_t*)d);
02943           tor_free(fp);
02944         });
02945       smartlist_free(fps);
02946     }
02947     if (!smartlist_len(dir_items) && !smartlist_len(items)) {
02948       write_http_status_line(conn, 404, "Not found");
02949       goto vote_done;
02950     }
02951     SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
02952                       body_len += compressed ? d->dir_z_len : d->dir_len);
02953     estimated_len += body_len;
02954     SMARTLIST_FOREACH(items, const char *, item, {
02955         size_t ln = strlen(item);
02956         if (compressed) {
02957           estimated_len += ln/2;
02958         } else {
02959           body_len += ln; estimated_len += ln;
02960         }
02961       });
02962 
02963     if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
02964       write_http_status_line(conn, 503, "Directory busy, try again later.");
02965       goto vote_done;
02966     }
02967     write_http_response_header(conn, body_len ? body_len : -1, compressed,
02968                  lifetime);
02969 
02970     if (smartlist_len(items)) {
02971       if (compressed) {
02972         conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
02973         SMARTLIST_FOREACH(items, const char *, c,
02974                  connection_write_to_buf_zlib(c, strlen(c), conn, 0));
02975         connection_write_to_buf_zlib("", 0, conn, 1);
02976       } else {
02977         SMARTLIST_FOREACH(items, const char *, c,
02978                          connection_write_to_buf(c, strlen(c), TO_CONN(conn)));
02979       }
02980     } else {
02981       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
02982           connection_write_to_buf(compressed ? d->dir_z : d->dir,
02983                                   compressed ? d->dir_z_len : d->dir_len,
02984                                   TO_CONN(conn)));
02985     }
02986   vote_done:
02987     smartlist_free(items);
02988     smartlist_free(dir_items);
02989     goto done;
02990   }
02991 
02992   if (!strcmpstart(url, "/tor/micro/d/")) {
02993     smartlist_t *fps = smartlist_new();
02994 
02995     dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"),
02996                                       fps, NULL,
02997                                       DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
02998 
02999     if (!dirserv_have_any_microdesc(fps)) {
03000       write_http_status_line(conn, 404, "Not found");
03001       SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
03002       smartlist_free(fps);
03003       goto done;
03004     }
03005     dlen = dirserv_estimate_microdesc_size(fps, compressed);
03006     if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
03007       log_info(LD_DIRSERV,
03008                "Client asked for server descriptors, but we've been "
03009                "writing too many bytes lately. Sending 503 Dir busy.");
03010       write_http_status_line(conn, 503, "Directory busy, try again later");
03011       SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
03012       smartlist_free(fps);
03013       goto done;
03014     }
03015 
03016     write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME);
03017     conn->dir_spool_src = DIR_SPOOL_MICRODESC;
03018     conn->fingerprint_stack = fps;
03019 
03020     if (compressed)
03021       conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
03022 
03023     connection_dirserv_flushed_some(conn);
03024     goto done;
03025   }
03026 
03027   if (!strcmpstart(url,"/tor/server/") ||
03028       (!options->BridgeAuthoritativeDir &&
03029        !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
03030     int res;
03031     const char *msg;
03032     const char *request_type = NULL;
03033     int cache_lifetime = 0;
03034     int is_extra = !strcmpstart(url,"/tor/extra/");
03035     url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
03036     conn->fingerprint_stack = smartlist_new();
03037     res = dirserv_get_routerdesc_fingerprints(conn->fingerprint_stack, url,
03038                                           &msg,
03039                                           !connection_dir_is_encrypted(conn),
03040                                           is_extra);
03041 
03042     if (!strcmpstart(url, "fp/")) {
03043       request_type = compressed?"/tor/server/fp.z":"/tor/server/fp";
03044       if (smartlist_len(conn->fingerprint_stack) == 1)
03045         cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
03046     } else if (!strcmpstart(url, "authority")) {
03047       request_type = compressed?"/tor/server/authority.z":
03048         "/tor/server/authority";
03049       cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
03050     } else if (!strcmpstart(url, "all")) {
03051       request_type = compressed?"/tor/server/all.z":"/tor/server/all";
03052       cache_lifetime = FULL_DIR_CACHE_LIFETIME;
03053     } else if (!strcmpstart(url, "d/")) {
03054       request_type = compressed?"/tor/server/d.z":"/tor/server/d";
03055       if (smartlist_len(conn->fingerprint_stack) == 1)
03056         cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
03057     } else {
03058       request_type = "/tor/server/?";
03059     }
03060     (void) request_type; /* usable for note_request. */
03061     if (!strcmpstart(url, "d/"))
03062       conn->dir_spool_src =
03063         is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
03064     else
03065       conn->dir_spool_src =
03066         is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
03067 
03068     if (!dirserv_have_any_serverdesc(conn->fingerprint_stack,
03069                                      conn->dir_spool_src)) {
03070       res = -1;
03071       msg = "Not found";
03072     }
03073 
03074     if (res < 0)
03075       write_http_status_line(conn, 404, msg);
03076     else {
03077       dlen = dirserv_estimate_data_size(conn->fingerprint_stack,
03078                                         1, compressed);
03079       if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
03080         log_info(LD_DIRSERV,
03081                  "Client asked for server descriptors, but we've been "
03082                  "writing too many bytes lately. Sending 503 Dir busy.");
03083         write_http_status_line(conn, 503, "Directory busy, try again later");
03084         conn->dir_spool_src = DIR_SPOOL_NONE;
03085         goto done;
03086       }
03087       write_http_response_header(conn, -1, compressed, cache_lifetime);
03088       if (compressed)
03089         conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
03090       /* Prime the connection with some data. */
03091       connection_dirserv_flushed_some(conn);
03092     }
03093     goto done;
03094   }
03095 
03096   if (!strcmpstart(url,"/tor/keys/")) {
03097     smartlist_t *certs = smartlist_new();
03098     ssize_t len = -1;
03099     if (!strcmp(url, "/tor/keys/all")) {
03100       authority_cert_get_all(certs);
03101     } else if (!strcmp(url, "/tor/keys/authority")) {
03102       authority_cert_t *cert = get_my_v3_authority_cert();
03103       if (cert)
03104         smartlist_add(certs, cert);
03105     } else if (!strcmpstart(url, "/tor/keys/fp/")) {
03106       smartlist_t *fps = smartlist_new();
03107       dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
03108                                            fps, NULL,
03109                                            DSR_HEX|DSR_SORT_UNIQ);
03110       SMARTLIST_FOREACH(fps, char *, d, {
03111           authority_cert_t *c = authority_cert_get_newest_by_id(d);
03112           if (c) smartlist_add(certs, c);
03113           tor_free(d);
03114       });
03115       smartlist_free(fps);
03116     } else if (!strcmpstart(url, "/tor/keys/sk/")) {
03117       smartlist_t *fps = smartlist_new();
03118       dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
03119                                            fps, NULL,
03120                                            DSR_HEX|DSR_SORT_UNIQ);
03121       SMARTLIST_FOREACH(fps, char *, d, {
03122           authority_cert_t *c = authority_cert_get_by_sk_digest(d);
03123           if (c) smartlist_add(certs, c);
03124           tor_free(d);
03125       });
03126       smartlist_free(fps);
03127     } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
03128       smartlist_t *fp_sks = smartlist_new();
03129       dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
03130                                                 fp_sks);
03131       SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
03132           authority_cert_t *c = authority_cert_get_by_digests(pair->first,
03133                                                               pair->second);
03134           if (c) smartlist_add(certs, c);
03135           tor_free(pair);
03136       });
03137       smartlist_free(fp_sks);
03138     } else {
03139       write_http_status_line(conn, 400, "Bad request");
03140       goto keys_done;
03141     }
03142     if (!smartlist_len(certs)) {
03143       write_http_status_line(conn, 404, "Not found");
03144       goto keys_done;
03145     }
03146     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
03147       if (c->cache_info.published_on < if_modified_since)
03148         SMARTLIST_DEL_CURRENT(certs, c));
03149     if (!smartlist_len(certs)) {
03150       write_http_status_line(conn, 304, "Not modified");
03151       goto keys_done;
03152     }
03153     len = 0;
03154     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
03155                       len += c->cache_info.signed_descriptor_len);
03156 
03157     if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) {
03158       write_http_status_line(conn, 503, "Directory busy, try again later.");
03159       goto keys_done;
03160     }
03161 
03162     write_http_response_header(conn, compressed?-1:len, compressed, 60*60);
03163     if (compressed) {
03164       conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
03165       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
03166             connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body,
03167                                          c->cache_info.signed_descriptor_len,
03168                                          conn, 0));
03169       connection_write_to_buf_zlib("", 0, conn, 1);
03170     } else {
03171       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
03172             connection_write_to_buf(c->cache_info.signed_descriptor_body,
03173                                     c->cache_info.signed_descriptor_len,
03174                                     TO_CONN(conn)));
03175     }
03176   keys_done:
03177     smartlist_free(certs);
03178     goto done;
03179   }
03180 
03181   if (options->HidServDirectoryV2 &&
03182        !strcmpstart(url,"/tor/rendezvous2/")) {
03183     /* Handle v2 rendezvous descriptor fetch request. */
03184     const char *descp;
03185     const char *query = url + strlen("/tor/rendezvous2/");
03186     if (strlen(query) == REND_DESC_ID_V2_LEN_BASE32) {
03187       log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
03188                safe_str(query));
03189       switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
03190         case 1: /* valid */
03191           write_http_response_header(conn, strlen(descp), 0, 0);
03192           connection_write_to_buf(descp, strlen(descp), TO_CONN(conn));
03193           break;
03194         case 0: /* well-formed but not present */
03195           write_http_status_line(conn, 404, "Not found");
03196           break;
03197         case -1: /* not well-formed */
03198           write_http_status_line(conn, 400, "Bad request");
03199           break;
03200       }
03201     } else { /* not well-formed */
03202       write_http_status_line(conn, 400, "Bad request");
03203     }
03204     goto done;
03205   }
03206 
03207   if (options->HSAuthoritativeDir && !strcmpstart(url,"/tor/rendezvous/")) {
03208     /* rendezvous descriptor fetch */
03209     const char *descp;
03210     size_t desc_len;
03211     const char *query = url+strlen("/tor/rendezvous/");
03212 
03213     log_info(LD_REND, "Handling rendezvous descriptor get");
03214     switch (rend_cache_lookup_desc(query, 0, &descp, &desc_len)) {
03215       case 1: /* valid */
03216         write_http_response_header_impl(conn, desc_len,
03217                                         "application/octet-stream",
03218                                         NULL, NULL, 0);
03219         note_request("/tor/rendezvous?/", desc_len);
03220         /* need to send descp separately, because it may include NULs */
03221         connection_write_to_buf(descp, desc_len, TO_CONN(conn));
03222         break;
03223       case 0: /* well-formed but not present */
03224         write_http_status_line(conn, 404, "Not found");
03225         break;
03226       case -1: /* not well-formed */
03227         write_http_status_line(conn, 400, "Bad request");
03228         break;
03229     }
03230     goto done;
03231   }
03232 
03233   if (options->BridgeAuthoritativeDir &&
03234       options->_BridgePassword_AuthDigest &&
03235       connection_dir_is_encrypted(conn) &&
03236       !strcmp(url,"/tor/networkstatus-bridges")) {
03237     char *status;
03238     char digest[DIGEST256_LEN];
03239 
03240     header = http_get_header(headers, "Authorization: Basic ");
03241     if (header)
03242       crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
03243 
03244     /* now make sure the password is there and right */
03245     if (!header ||
03246         tor_memneq(digest,
03247                    options->_BridgePassword_AuthDigest, DIGEST256_LEN)) {
03248       write_http_status_line(conn, 404, "Not found");
03249       tor_free(header);
03250       goto done;
03251     }
03252     tor_free(header);
03253 
03254     /* all happy now. send an answer. */
03255     status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
03256     dlen = strlen(status);
03257     write_http_response_header(conn, dlen, 0, 0);
03258     connection_write_to_buf(status, dlen, TO_CONN(conn));
03259     tor_free(status);
03260     goto done;
03261   }
03262 
03263   if (!strcmpstart(url,"/tor/bytes.txt")) {
03264     char *bytes = directory_dump_request_log();
03265     size_t len = strlen(bytes);
03266     write_http_response_header(conn, len, 0, 0);
03267     connection_write_to_buf(bytes, len, TO_CONN(conn));
03268     tor_free(bytes);
03269     goto done;
03270   }
03271 
03272   if (!strcmp(url,"/tor/robots.txt")) { /* /robots.txt will have been
03273                                            rewritten to /tor/robots.txt */
03274     char robots[] = "User-agent: *\r\nDisallow: /\r\n";
03275     size_t len = strlen(robots);
03276     write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME);
03277     connection_write_to_buf(robots, len, TO_CONN(conn));
03278     goto done;
03279   }
03280 
03281   if (!strcmp(url,"/tor/dbg-stability.txt")) {
03282     const char *stability;
03283     size_t len;
03284     if (options->BridgeAuthoritativeDir ||
03285         ! authdir_mode_tests_reachability(options) ||
03286         ! (stability = rep_hist_get_router_stability_doc(time(NULL)))) {
03287       write_http_status_line(conn, 404, "Not found.");
03288       goto done;
03289     }
03290 
03291     len = strlen(stability);
03292     write_http_response_header(conn, len, 0, 0);
03293     connection_write_to_buf(stability, len, TO_CONN(conn));
03294     goto done;
03295   }
03296 
03297 #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
03298 #define ADD_MALLINFO_LINE(x) do {                               \
03299     smartlist_add_asprintf(lines, "%s %d\n", #x, mi.x);        \
03300   }while(0);
03301 
03302   if (!strcmp(url,"/tor/mallinfo.txt") &&
03303       (tor_addr_eq_ipv4h(&conn->_base.addr, 0x7f000001ul))) {
03304     char *result;
03305     size_t len;
03306     struct mallinfo mi;
03307     smartlist_t *lines;
03308 
03309     memset(&mi, 0, sizeof(mi));
03310     mi = mallinfo();
03311     lines = smartlist_new();
03312 
03313     ADD_MALLINFO_LINE(arena)
03314     ADD_MALLINFO_LINE(ordblks)
03315     ADD_MALLINFO_LINE(smblks)
03316     ADD_MALLINFO_LINE(hblks)
03317     ADD_MALLINFO_LINE(hblkhd)
03318     ADD_MALLINFO_LINE(usmblks)
03319     ADD_MALLINFO_LINE(fsmblks)
03320     ADD_MALLINFO_LINE(uordblks)
03321     ADD_MALLINFO_LINE(fordblks)
03322     ADD_MALLINFO_LINE(keepcost)
03323 
03324     result = smartlist_join_strings(lines, "", 0, NULL);
03325     SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
03326     smartlist_free(lines);
03327 
03328     len = strlen(result);
03329     write_http_response_header(conn, len, 0, 0);
03330     connection_write_to_buf(result, len, TO_CONN(conn));
03331     tor_free(result);
03332     goto done;
03333   }
03334 #endif
03335 
03336   /* we didn't recognize the url */
03337   write_http_status_line(conn, 404, "Not found");
03338 
03339  done:
03340   tor_free(url_mem);
03341   return 0;
03342 }
03343 
03349 static int
03350 directory_handle_command_post(dir_connection_t *conn, const char *headers,
03351                               const char *body, size_t body_len)
03352 {
03353   char *url = NULL;
03354   const or_options_t *options = get_options();
03355 
03356   log_debug(LD_DIRSERV,"Received POST command.");
03357 
03358   conn->_base.state = DIR_CONN_STATE_SERVER_WRITING;
03359 
03360   if (parse_http_url(headers, &url) < 0) {
03361     write_http_status_line(conn, 400, "Bad request");
03362     return 0;
03363   }
03364   log_debug(LD_DIRSERV,"rewritten url as '%s'.", url);
03365 
03366   /* Handle v2 rendezvous service publish request. */
03367   if (options->HidServDirectoryV2 &&
03368       !strcmpstart(url,"/tor/rendezvous2/publish")) {
03369     switch (rend_cache_store_v2_desc_as_dir(body)) {
03370       case -2:
03371         log_info(LD_REND, "Rejected v2 rend descriptor (length %d) from %s "
03372                  "since we're not currently a hidden service directory.",
03373                  (int)body_len, conn->_base.address);
03374         write_http_status_line(conn, 503, "Currently not acting as v2 "
03375                                "hidden service directory");
03376         break;
03377       case -1:
03378         log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.",
03379                  (int)body_len, conn->_base.address);
03380         write_http_status_line(conn, 400,
03381                                "Invalid v2 service descriptor rejected");
03382         break;
03383       default:
03384         write_http_status_line(conn, 200, "Service descriptor (v2) stored");
03385         log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
03386     }
03387     goto done;
03388   }
03389 
03390   if (!authdir_mode(options)) {
03391     /* we just provide cached directories; we don't want to
03392      * receive anything. */
03393     write_http_status_line(conn, 400, "Nonauthoritative directory does not "
03394                            "accept posted server descriptors");
03395     goto done;
03396   }
03397 
03398   if (authdir_mode_handles_descs(options, -1) &&
03399       !strcmp(url,"/tor/")) { /* server descriptor post */
03400     const char *msg = "[None]";
03401     uint8_t purpose = authdir_mode_bridge(options) ?
03402                       ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
03403     was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
03404                                              conn->_base.address, &msg);
03405     tor_assert(msg);
03406     if (WRA_WAS_ADDED(r))
03407       dirserv_get_directory(); /* rebuild and write to disk */
03408 
03409     if (r == ROUTER_ADDED_NOTIFY_GENERATOR) {
03410       /* Accepted with a message. */
03411       log_info(LD_DIRSERV,
03412                "Problematic router descriptor or extra-info from %s "
03413                "(\"%s\").",
03414                conn->_base.address, msg);
03415       write_http_status_line(conn, 400, msg);
03416     } else if (r == ROUTER_ADDED_SUCCESSFULLY) {
03417       write_http_status_line(conn, 200, msg);
03418     } else if (WRA_WAS_OUTDATED(r)) {
03419       write_http_response_header_impl(conn, -1, NULL, NULL,
03420                                       "X-Descriptor-Not-New: Yes\r\n", -1);
03421     } else {
03422       log_info(LD_DIRSERV,
03423                "Rejected router descriptor or extra-info from %s "
03424                "(\"%s\").",
03425                conn->_base.address, msg);
03426       write_http_status_line(conn, 400, msg);
03427     }
03428     goto done;
03429   }
03430 
03431   if (options->HSAuthoritativeDir &&
03432       !strcmpstart(url,"/tor/rendezvous/publish")) {
03433     /* rendezvous descriptor post */
03434     log_info(LD_REND, "Handling rendezvous descriptor post.");
03435     if (rend_cache_store(body, body_len, 1, NULL) < 0) {
03436       log_fn(LOG_PROTOCOL_WARN, LD_DIRSERV,
03437              "Rejected rend descriptor (length %d) from %s.",
03438              (int)body_len, conn->_base.address);
03439       write_http_status_line(conn, 400,
03440                              "Invalid v0 service descriptor rejected");
03441     } else {
03442       write_http_status_line(conn, 200, "Service descriptor (v0) stored");
03443     }
03444     goto done;
03445   }
03446 
03447   if (authdir_mode_v3(options) &&
03448       !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
03449     const char *msg = "OK";
03450     int status;
03451     if (dirvote_add_vote(body, &msg, &status)) {
03452       write_http_status_line(conn, status, "Vote stored");
03453     } else {
03454       tor_assert(msg);
03455       log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
03456                conn->_base.address, msg);
03457       write_http_status_line(conn, status, msg);
03458     }
03459     goto done;
03460   }
03461 
03462   if (authdir_mode_v3(options) &&
03463       !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
03464     const char *msg = NULL;
03465     if (dirvote_add_signatures(body, conn->_base.address, &msg)>=0) {
03466       write_http_status_line(conn, 200, msg?msg:"Signatures stored");
03467     } else {
03468       log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
03469                conn->_base.address, msg?msg:"???");
03470       write_http_status_line(conn, 400, msg?msg:"Unable to store signatures");
03471     }
03472     goto done;
03473   }
03474 
03475   /* we didn't recognize the url */
03476   write_http_status_line(conn, 404, "Not found");
03477 
03478  done:
03479   tor_free(url);
03480   return 0;
03481 }
03482 
03488 static int
03489 directory_handle_command(dir_connection_t *conn)
03490 {
03491   char *headers=NULL, *body=NULL;
03492   size_t body_len=0;
03493   int r;
03494 
03495   tor_assert(conn);
03496   tor_assert(conn->_base.type == CONN_TYPE_DIR);
03497 
03498   switch (connection_fetch_from_buf_http(TO_CONN(conn),
03499                               &headers, MAX_HEADERS_SIZE,
03500                               &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
03501     case -1: /* overflow */
03502       log_warn(LD_DIRSERV,
03503                "Request too large from address '%s' to DirPort. Closing.",
03504                safe_str(conn->_base.address));
03505       return -1;
03506     case 0:
03507       log_debug(LD_DIRSERV,"command not all here yet.");
03508       return 0;
03509     /* case 1, fall through */
03510   }
03511 
03512   http_set_address_origin(headers, TO_CONN(conn));
03513   //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, body);
03514 
03515   if (!strncasecmp(headers,"GET",3))
03516     r = directory_handle_command_get(conn, headers, body, body_len);
03517   else if (!strncasecmp(headers,"POST",4))
03518     r = directory_handle_command_post(conn, headers, body, body_len);
03519   else {
03520     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
03521            "Got headers %s with unknown command. Closing.",
03522            escaped(headers));
03523     r = -1;
03524   }
03525 
03526   tor_free(headers); tor_free(body);
03527   return r;
03528 }
03529 
03534 int
03535 connection_dir_finished_flushing(dir_connection_t *conn)
03536 {
03537   tor_assert(conn);
03538   tor_assert(conn->_base.type == CONN_TYPE_DIR);
03539 
03540   /* Note that we have finished writing the directory response. For direct
03541    * connections this means we're done, for tunneled connections its only
03542    * an intermediate step. */
03543   if (TO_CONN(conn)->dirreq_id)
03544     geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED,
03545                               DIRREQ_FLUSHING_DIR_CONN_FINISHED);
03546   else
03547     geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
03548                               DIRREQ_DIRECT,
03549                               DIRREQ_FLUSHING_DIR_CONN_FINISHED);
03550   switch (conn->_base.state) {
03551     case DIR_CONN_STATE_CONNECTING:
03552     case DIR_CONN_STATE_CLIENT_SENDING:
03553       log_debug(LD_DIR,"client finished sending command.");
03554       conn->_base.state = DIR_CONN_STATE_CLIENT_READING;
03555       return 0;
03556     case DIR_CONN_STATE_SERVER_WRITING:
03557       if (conn->dir_spool_src != DIR_SPOOL_NONE) {
03558 #ifdef USE_BUFFEREVENTS
03559         /* This can happen with paired bufferevents, since a paired connection
03560          * can flush immediately when you write to it, making the subsequent
03561          * check in connection_handle_write_cb() decide that the connection
03562          * is flushed. */
03563         log_debug(LD_DIRSERV, "Emptied a dirserv buffer, but still spooling.");
03564 #else
03565         log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
03566         connection_mark_for_close(TO_CONN(conn));
03567 #endif
03568       } else {
03569         log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
03570         connection_mark_for_close(TO_CONN(conn));
03571       }
03572       return 0;
03573     default:
03574       log_warn(LD_BUG,"called in unexpected state %d.",
03575                conn->_base.state);
03576       tor_fragile_assert();
03577       return -1;
03578   }
03579   return 0;
03580 }
03581 
03584 int
03585 connection_dir_finished_connecting(dir_connection_t *conn)
03586 {
03587   tor_assert(conn);
03588   tor_assert(conn->_base.type == CONN_TYPE_DIR);
03589   tor_assert(conn->_base.state == DIR_CONN_STATE_CONNECTING);
03590 
03591   log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
03592             conn->_base.address,conn->_base.port);
03593 
03594   conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING; /* start flushing conn */
03595   return 0;
03596 }
03597 
03601 static void
03602 dir_networkstatus_download_failed(smartlist_t *failed, int status_code)
03603 {
03604   if (status_code == 503)
03605     return;
03606   SMARTLIST_FOREACH(failed, const char *, fp,
03607   {
03608     char digest[DIGEST_LEN];
03609     trusted_dir_server_t *dir;
03610     if (base16_decode(digest, DIGEST_LEN, fp, strlen(fp))<0) {
03611       log_warn(LD_BUG, "Called with bad fingerprint in list: %s",
03612                escaped(fp));
03613       continue;
03614     }
03615     dir = router_get_trusteddirserver_by_digest(digest);
03616 
03617     if (dir)
03618       download_status_failed(&dir->v2_ns_dl_status, status_code);
03619   });
03620 }
03621 
03623 static const int server_dl_schedule[] = {
03624   0, 0, 0, 60, 60, 60*2, 60*5, 60*15, INT_MAX
03625 };
03627 static const int client_dl_schedule[] = {
03628   0, 0, 60, 60*5, 60*10, INT_MAX
03629 };
03631 static const int server_consensus_dl_schedule[] = {
03632   0, 0, 60, 60*5, 60*10, 60*30, 60*30, 60*30, 60*30, 60*30, 60*60, 60*60*2
03633 };
03635 static const int client_consensus_dl_schedule[] = {
03636   0, 0, 60, 60*5, 60*10, 60*30, 60*60, 60*60, 60*60, 60*60*3, 60*60*6, 60*60*12
03637 };
03639 static const int bridge_dl_schedule[] = {
03640   60*60, 15*60, 15*60, 60*60
03641 };
03642 
03646 static void
03647 find_dl_schedule_and_len(download_status_t *dls, int server,
03648                          const int **schedule, size_t *schedule_len)
03649 {
03650   switch (dls->schedule) {
03651     case DL_SCHED_GENERIC:
03652       if (server) {
03653         *schedule = server_dl_schedule;
03654         *schedule_len = sizeof(server_dl_schedule)/sizeof(int);
03655       } else {
03656         *schedule = client_dl_schedule;
03657         *schedule_len = sizeof(client_dl_schedule)/sizeof(int);
03658       }
03659       break;
03660     case DL_SCHED_CONSENSUS:
03661       if (server) {
03662         *schedule = server_consensus_dl_schedule;
03663         *schedule_len = sizeof(server_consensus_dl_schedule)/sizeof(int);
03664       } else {
03665         *schedule = client_consensus_dl_schedule;
03666         *schedule_len = sizeof(client_consensus_dl_schedule)/sizeof(int);
03667       }
03668       break;
03669     case DL_SCHED_BRIDGE:
03670       *schedule = bridge_dl_schedule;
03671       *schedule_len = sizeof(bridge_dl_schedule)/sizeof(int);
03672       break;
03673     default:
03674       tor_assert(0);
03675   }
03676 }
03677 
03682 time_t
03683 download_status_increment_failure(download_status_t *dls, int status_code,
03684                                   const char *item, int server, time_t now)
03685 {
03686   const int *schedule;
03687   size_t schedule_len;
03688   int increment;
03689   tor_assert(dls);
03690   if (status_code != 503 || server) {
03691     if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1)
03692       ++dls->n_download_failures;
03693   }
03694 
03695   find_dl_schedule_and_len(dls, server, &schedule, &schedule_len);
03696 
03697   if (dls->n_download_failures < schedule_len)
03698     increment = schedule[dls->n_download_failures];
03699   else if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD)
03700     increment = INT_MAX;
03701   else
03702     increment = schedule[schedule_len-1];
03703 
03704   if (increment < INT_MAX)
03705     dls->next_attempt_at = now+increment;
03706   else
03707     dls->next_attempt_at = TIME_MAX;
03708 
03709   if (item) {
03710     if (increment == 0)
03711       log_debug(LD_DIR, "%s failed %d time(s); I'll try again immediately.",
03712                 item, (int)dls->n_download_failures);
03713     else if (dls->next_attempt_at < TIME_MAX)
03714       log_debug(LD_DIR, "%s failed %d time(s); I'll try again in %d seconds.",
03715                 item, (int)dls->n_download_failures,
03716                 (int)(dls->next_attempt_at-now));
03717     else
03718       log_debug(LD_DIR, "%s failed %d time(s); Giving up for a while.",
03719                 item, (int)dls->n_download_failures);
03720   }
03721   return dls->next_attempt_at;
03722 }
03723 
03732 void
03733 download_status_reset(download_status_t *dls)
03734 {
03735   const int *schedule;
03736   size_t schedule_len;
03737 
03738   find_dl_schedule_and_len(dls, get_options()->DirPort != NULL,
03739                            &schedule, &schedule_len);
03740 
03741   dls->n_download_failures = 0;
03742   dls->next_attempt_at = time(NULL) + schedule[0];
03743 }
03744 
03747 int
03748 download_status_get_n_failures(const download_status_t *dls)
03749 {
03750   return dls->n_download_failures;
03751 }
03752 
03758 static void
03759 dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
03760                                int router_purpose,
03761                                int was_extrainfo, int was_descriptor_digests)
03762 {
03763   char digest[DIGEST_LEN];
03764   time_t now = time(NULL);
03765   int server = directory_fetches_from_authorities(get_options());
03766   if (!was_descriptor_digests) {
03767     if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
03768       tor_assert(!was_extrainfo);
03769       connection_dir_retry_bridges(failed);
03770     }
03771     return; /* FFFF should implement for other-than-router-purpose someday */
03772   }
03773   SMARTLIST_FOREACH(failed, const char *, cp,
03774   {
03775     download_status_t *dls = NULL;
03776     if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) < 0) {
03777       log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp));
03778       continue;
03779     }
03780     if (was_extrainfo) {
03781       signed_descriptor_t *sd =
03782         router_get_by_extrainfo_digest(digest);
03783       if (sd)
03784         dls = &sd->ei_dl_status;
03785     } else {
03786       dls = router_get_dl_status_by_descriptor_digest(digest);
03787     }
03788     if (!dls || dls->n_download_failures >= MAX_ROUTERDESC_DOWNLOAD_FAILURES)
03789       continue;
03790     download_status_increment_failure(dls, status_code, cp, server, now);
03791   });
03792 
03793   /* No need to relaunch descriptor downloads here: we already do it
03794    * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
03795 }
03796 
03801 static void
03802 dir_microdesc_download_failed(smartlist_t *failed,
03803                               int status_code)
03804 {
03805   networkstatus_t *consensus
03806     = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
03807   routerstatus_t *rs;
03808   download_status_t *dls;
03809   time_t now = time(NULL);
03810   int server = directory_fetches_from_authorities(get_options());
03811 
03812   if (! consensus)
03813     return;
03814   SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
03815     rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
03816     if (!rs)
03817       continue;
03818     dls = &rs->dl_status;
03819     if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES)
03820       continue;
03821     {
03822       char buf[BASE64_DIGEST256_LEN+1];
03823       digest256_to_base64(buf, d);
03824       download_status_increment_failure(dls, status_code, buf,
03825                                         server, now);
03826     }
03827   } SMARTLIST_FOREACH_END(d);
03828 }
03829 
03832 static int
03833 _compare_pairs(const void **a, const void **b)
03834 {
03835   const fp_pair_t *fp1 = *a, *fp2 = *b;
03836   int r;
03837   if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
03838     return r;
03839   else
03840     return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
03841 }
03842 
03847 int
03848 dir_split_resource_into_fingerprint_pairs(const char *res,
03849                                           smartlist_t *pairs_out)
03850 {
03851   smartlist_t *pairs_tmp = smartlist_new();
03852   smartlist_t *pairs_result = smartlist_new();
03853 
03854   smartlist_split_string(pairs_tmp, res, "+", 0, 0);
03855   if (smartlist_len(pairs_tmp)) {
03856     char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
03857     size_t last_len = strlen(last);
03858     if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
03859       last[last_len-2] = '\0';
03860     }
03861   }
03862   SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
03863     if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
03864       log_info(LD_DIR,
03865              "Skipping digest pair %s with non-standard length.", escaped(cp));
03866     } else if (cp[HEX_DIGEST_LEN] != '-') {
03867       log_info(LD_DIR,
03868              "Skipping digest pair %s with missing dash.", escaped(cp));
03869     } else {
03870       fp_pair_t pair;
03871       if (base16_decode(pair.first, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0 ||
03872           base16_decode(pair.second,
03873                         DIGEST_LEN, cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN)<0) {
03874         log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
03875       } else {
03876         smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
03877       }
03878     }
03879     tor_free(cp);
03880   } SMARTLIST_FOREACH_END(cp);
03881   smartlist_free(pairs_tmp);
03882 
03883   /* Uniq-and-sort */
03884   smartlist_sort(pairs_result, _compare_pairs);
03885   smartlist_uniq(pairs_result, _compare_pairs, _tor_free);
03886 
03887   smartlist_add_all(pairs_out, pairs_result);
03888   smartlist_free(pairs_result);
03889   return 0;
03890 }
03891 
03905 int
03906 dir_split_resource_into_fingerprints(const char *resource,
03907                                      smartlist_t *fp_out, int *compressed_out,
03908                                      int flags)
03909 {
03910   const int decode_hex = flags & DSR_HEX;
03911   const int decode_base64 = flags & DSR_BASE64;
03912   const int digests_are_256 = flags & DSR_DIGEST256;
03913   const int sort_uniq = flags & DSR_SORT_UNIQ;
03914 
03915   const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
03916   const int hex_digest_len = digests_are_256 ?
03917     HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
03918   const int base64_digest_len = digests_are_256 ?
03919     BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
03920   smartlist_t *fp_tmp = smartlist_new();
03921 
03922   tor_assert(!(decode_hex && decode_base64));
03923   tor_assert(fp_out);
03924 
03925   smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
03926   if (compressed_out)
03927     *compressed_out = 0;
03928   if (smartlist_len(fp_tmp)) {
03929     char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
03930     size_t last_len = strlen(last);
03931     if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
03932       last[last_len-2] = '\0';
03933       if (compressed_out)
03934         *compressed_out = 1;
03935     }
03936   }
03937   if (decode_hex || decode_base64) {
03938     const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
03939     int i;
03940     char *cp, *d = NULL;
03941     for (i = 0; i < smartlist_len(fp_tmp); ++i) {
03942       cp = smartlist_get(fp_tmp, i);
03943       if (strlen(cp) != encoded_len) {
03944         log_info(LD_DIR,
03945                  "Skipping digest %s with non-standard length.", escaped(cp));
03946         smartlist_del_keeporder(fp_tmp, i--);
03947         goto again;
03948       }
03949       d = tor_malloc_zero(digest_len);
03950       if (decode_hex ?
03951           (base16_decode(d, digest_len, cp, hex_digest_len)<0) :
03952           (base64_decode(d, digest_len, cp, base64_digest_len)<0)) {
03953           log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
03954           smartlist_del_keeporder(fp_tmp, i--);
03955           goto again;
03956       }
03957       smartlist_set(fp_tmp, i, d);
03958       d = NULL;
03959     again:
03960       tor_free(cp);
03961       tor_free(d);
03962     }
03963   }
03964   if (sort_uniq) {
03965     if (decode_hex || decode_base64) {
03966       if (digests_are_256) {
03967         smartlist_sort_digests256(fp_tmp);
03968         smartlist_uniq_digests256(fp_tmp);
03969       } else {
03970         smartlist_sort_digests(fp_tmp);
03971         smartlist_uniq_digests(fp_tmp);
03972       }
03973     } else {
03974       smartlist_sort_strings(fp_tmp);
03975       smartlist_uniq_strings(fp_tmp);
03976     }
03977   }
03978   smartlist_add_all(fp_out, fp_tmp);
03979   smartlist_free(fp_tmp);
03980   return 0;
03981 }
03982