܍~coddlZddlZddlZddlZddlZddlmZmZmZddl m Z ddl m Z ddl mZddlmZddlmZddlmZmZmZmZmZmZmZdd lmZmZmZddlZdd lm Z dd l!m"Z#dd l!m$Z$e#j%e&Z'd Z(dZ)dZ*dedfdZ+GddZ,Gdde,Z-GddZ.Gdde/Z0dZ1 dd"ed#efd+ee4d,e7d%e5def d-Z9dde'j:dddddddd)f d.ed/ed0e5d1ed2eee5ge5fd3e4d4e2d5e7fd6Z;Gd7d8Z< d=d9Z=efej>fd:d;Z?dS)?N)ThreadPoolExecutor TimeoutError as_completed) parsedate)ENOENT)partial) NOT_FOUND)count)AnyCallableIteratorListOptionalTupleUnion)quoteurlparse urlunparse) exceptions)log)versionREDACTEDctt|d}|ds|dr|d|d<d|d<t|S)Nhttp)scheme)listrr)url parsed_urls 6/usr/lib/python3/dist-packages/cloudinit/url_helper.py _cleanurlr#$sYhs622233J a=Z]#1 1  1 j ! !!c2d}|}|D]}|||}|S)Nctt|}|d}|r|ds|dz }|tt |dz }||d<t |S)Nr/z/:)safe)rrendswithrstrr)r add_on url_parsedpaths r"combine_singlez#combine_url..combine_single/sv(3--(( !}   c**  CKD c&kk---- 1 *%%%r$)baseadd_onsr.r r+s r" combine_urlr2.s?&&& C**nS&)) Jr$return) FileResponse UrlResponsec z|}|drd|z}|dr|drtd||t dd} t|d5}|}dddn #1swxYwYnD#t$r7}|j }|j tkrt}t||d||d}~wwxYwt|| St|fi|S) a0Wrapper function around readurl to allow passing a file path as url. When url is not a local file path, passthrough any kwargs to readurl. In the case of parameter passthrough to readurl, default values for some parameters. See: call-signature of readurl in this module for param docs. r'z file://%szfile://dataz'Unable to post data to file resource %sNrb)causecodeheadersr )contents)lstrip startswithlowergetLOGwarninglenopenreadIOErrorerrnorr UrlErrorr4readurl)r kwargs file_pathfpr<er:s r"read_file_or_urlrN>s **,,C ~~c C yy{{i((& ::f   H KKA3 G G GI(()  Mi&& %"7799 % % % % % % % % % % % % % % % M M M7Dw&   tEEE1 L  M I9999s%%f%%%s<C.C CCCCC D%2DDc"eZdZddZdZdZdS)StringResponsec>||_i|_||_d|_dSN)r:r;r<r )selfr<r:s r"__init__zStringResponse.__init__^s"    r$c |jdkrdSdS)NrQFTr:rTargsrJs r"okzStringResponse.okds 9  5tr$c6|jdS)Nzutf-8)r<decoderTs r"__str__zStringResponse.__str__is}##G,,,r$NrQ)__name__ __module__ __qualname__rUrZr^r/r$r"rPrP]sF  -----r$rPceZdZddZdS)r4rQcNt|||||_dS)NrW)rPrUr )rTr-r<r:s r"rUzFileResponse.__init__ns(hT:::r$Nr_r`rarbrUr/r$r"r4r4ms(r$r4ceZdZdejfdZedefdZedZ ddZ edZ ed Z d Z dd eed edeefdZdS)r5responsec||_dSrS) _response)rTrgs r"rUzUrlResponse.__init__ts !r$r3c6|jjdS|jjS)Nr$)ricontentr]s r"r<zUrlResponse.contentsws > ! )3~%%r$c|jjSrS)rir r]s r"r zUrlResponse.url}s ~!!r$Fc@d}|rd}d|jcxkr|krnndSdS)Ni,irQTFrW)rT redirects_okuppers r"rZzUrlResponse.oksB  E $) # # # #e # # # # #45r$c|jjSrS)rir;r]s r"r;zUrlResponse.headerss ~%%r$c|jjSrS)ri status_coder]s r"r:zUrlResponse.codes ~))r$c|jjSrS)ritextr]s r"r^zUrlResponse.__str__s ~""r$r chunk_sizedecode_unicodec#LK|j||Ed{VdS)atIterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. :param chunk_size: Number of bytes it should read into memory. :param decode_unicode: If True, content will be decoded using the best available encoding based on the response. N)ri iter_content)rTrurvs r"rxzUrlResponse.iter_contents8>..z>JJJJJJJJJJJr$N)F)rF)r`rarbrequestsResponserUpropertybytesr<r rZr;r:r^rintboolr rxr/r$r"r5r5ss"!2""""&%&&&X& ""X"&&X&**X*###EJ K K"3- K=A K % K K K K K Kr$r5ceZdZddZdS)rHNct|t|||_||_||_|ji|_||_dSrS)rFrUr*r9r:r;r )rTr9r:r;r s r"rUzUrlError.__init__sMs5zz***   < DLr$)NNNrer/r$r"rHrHs(r$rHci}t|j}|dkrW|rUd|vr|dr |d|d<nd|d<d|vrd|vr|d|dg|d<nd|vrt|d|d<|S)Nhttpsca_certsverifyT cert_filekey_filecert)rrr*)r ssl_detailsssl_argsrs r" _get_ssl_argsrsH c]] !F [  $ $Z)@ $!,Z!8HX  !%HX  + % %* *C*CK(J' HV  K ' '";{#;<._cbsNr$r7r;c>g|]}||Sr/)r@).0kvs r" zreadurl..'s("I"I"Ia"I1"I"I"Ir$z'[%s/%s] open '%s' with %s configurationinfinitez(Read from %s (%s, %sb) after %s attemptsrgrrr:r;r )r 1Please wait %s seconds while we wait to try againr/)%r#updater isinstancetuplemaxfloatr}rversion_stringr rangeitemscopydeepcopyrrAdebugrySessionrequestraise_for_statusrrrCrkr5rRequestException HTTPErrorhasattrrgappendrHr;SSLErrortimesleep)r r7rretries sec_betweenr; headers_cbheaders_redactr check_statusr exception_cbsessionr log_req_resprequest_methodrreq_args manual_tries def_headersrexcpsifiltered_req_argsrmatched_headerskeysessrrMrs ` @r"rIrIsh C..CH OOM#{33444"1H  3#'2U'HX gu % % 9")HY  "%eGnna"8"8HY L03w<> ! ! !*$ ($ ($ (1z355 Az** J   Z3 ! 2  XaS111222a!455EEEEE LL%)$L$L  ([1__A $$q IIK# ;'''I$ (L )OsK;IG, I,G0 0I3G0 4AIN#&B8N$&NANN#func.addrreventdelaycN|r||rdS|||S)z Execute func with optional delayrNwaitrrrrrs r"_run_func_with_delayrus;  ::e: $ $  F 4g  r$333333? addresses stagger_delaycx d}d}d}g}tj tt| fdt |D}t |D]y} || }| } | r| }|j|3| }|r0 ||fc dSz|rt d|||t d|td #t$rEt d d |d |YnwxYw dn# dwxYw||fS) aexecute multiple callbacks in parallel Run blocking func against two different addresses staggered with a delay. The first call to return successfully is returned from this function and remaining unfinished calls are cancelled if they have not yet started N) max_workersc ^i|])\}}t||z|*S)r)submitr)rrrexecutorris_donerrs r" zdual_stack..s^    4 OO$=(       r$rFrz.default_sleep_times;?##a''r$cJ|dS|dkptj|z |kS)z4Check if time is up based on start time and max waitNFr)r)max_wait start_times r"timeupzwait_for_url..timeups,  5A G49;;#;h#FGr$c|js5d|jz}tt||j|j|}nM|s5d|jz}tt||j|j|}nd}d}||fS)z?Map requests response code/contents to internal "UrlError" typezempty response [%s]rzbad status code [%s]rN)r<r:rHrr;rZ)rgr reasonurl_excs r"handle_url_responsez)wait_for_url..handle_url_responses  *hm.read_url_handle_exceptions.sM )M$//MC11(C@@OGV 'X& '   )A-FGGGGGG   ,q0FGGGGGG z122 +3Dux'' CC JJ LL F  z  6 FzW 5 5 5 5 5 5  6 6s "+ AA AAAcHt|in ||dS)NF)r;rrrr)rI)r rrrrs r" read_url_cbz!wait_for_url..read_url_cbNs= $,BB**S//))     r$c fd} D]d}tj}dkr4 |rdS #r!|z| zkrt| z|z  |||||}|r|cSedS)z|iterate over list of urls, request each one and handle responses and thrown exceptions individually per url c ||fSrSr/)r r rs r"url_reader_serialz@wait_for_url..read_url_serial..url_reader_serial]sS'223 3r$rN)rr})rrrrrr nowoutloop_nrr r rrs ` r"read_url_serialz%wait_for_url..read_url_serialXs  4 4 4 4 4 4  C)++C{{6(J//FF()w*x*?@@":#8C"?@@G,,!3 FFC   !  r$c^tt|}| |||}|r|SdS)zpass list of urls to dual_stack which sends requests in parallel handle response and exceptions of the first endpoint to respond )rrN)rr) rrrrurl_reader_parallelrrr r rs r"read_url_parallelz'wait_for_url..read_url_parallelss^&  %     )( z66    J  r$rNTrrz$Timed out, no response from urls: %s)FN)r}rr<rArrr)rrrrrrrrrrrrrrrr do_read_urlcalculate_sleep_timergr addressrrr r rs`` `` ` ` @@@@@r" wait_for_urlrsh(3(3((((HHH !!!.666666@       6 J1G6G)>,>FHE))(F;; k*g|YGG  0 # GXX./ / 6(J ' '  ! ?    :zH, ;<<D'E*II4d;;; ;r$cVeZdZ d dZdZdZdZdZdZd Z d Z d Z d Z dS)OauthUrlHelperN/run/oauth_skew.jsonc8||_|pd|_||_||_||_d|_d|_|j|j|jf}t|sd|_nt|std| }|pi|_ dS)NrTrFzBall or none of token_key, token_secret, or consumer_key can be set) consumer_keyconsumer_secret token_key token_secretskew_data_file _do_oauthskew_change_limitanyallrread_skew_file skew_data)rTrr!r"r r#requiredolds r"rUzOauthUrlHelper.__init__s).4""(,!"ND$5t7HI8}} "DNNX *  !!##r$c|jrftj|jrBt |jd5}t j|cdddS#1swxYwYdS)Nrmode)r#osr-isfilerDjsonload)rTrLs r"r(zOauthUrlHelper.read_skew_files   %27>>$2E#F#F %d)444 %y}} % % % % % % % % % % % % % % % %tsA##A'*A'c|jsdS|}|i}|||<t|jd5}|t j|ddddS#1swxYwYdS)Nwr-)r#r(rDwriter1dumps)rThostvaluecurrLs r"update_skew_filezOauthUrlHelper.update_skew_files"  F!!## ;CD $%C 0 0 0 &B HHTZ__ % % % & & & & & & & & & & & & & & & & & &s(A22A69A6ct|tr|jdks |jdksdSd|jvr"td|jdS|jd} t jt|}n4#t$r'}td||Yd}~dSd}~wwxYwt|t jz }t|j j }|j|d}t!||z |jkr2|||td||||j|<dS)Niidatez$Missing header 'date' in %s responsez#Failed to convert datetime '%s': %srz$Setting oauth clockskew for %s to %d)rrHr:r;rArBrmktimerrr}rr netlocr)r@absr%r:) rTrrr< remote_timerMskewr7old_skews r"rzOauthUrlHelper.exception_cbsc y( + + 3&&).C*?*? F * * * KK>  O O O F ( +ioo66KK    KK=tQ G G G FFFFF ;,-- &&->%%dA.. x$  $"8 8 8  ! !$ - - - KK>d K K K#ts'!B B:B55B:c|jsiSd}t|j}|jr7||jvr.t t j|j|z}t ||j|j|j |j |S)N)r rr!r"r  timestamp) r$rr>r)r}r oauth_headersrr!r"r )rTr rDr7s r"rzOauthUrlHelper.headers_cbs~ I }}# > @ddn44DIKK((4>$+??I*n* 0     r$ct|j|d|d<t|j|d|d<||i|S)Nrr)r _headers_cbr@ _exception_cb)rT wrapped_funcrYrJs r"_wrappedzOauthUrlHelper._wrappedsk&  fjj66  |")   > : :" " ~|T,V,,,r$c:|t||SrS)rJrrXs r"rzOauthUrlHelper.wait_for_urls}}\4888r$c:|t||SrS)rJrIrXs r"rIzOauthUrlHelper.readurls}}WdF333r$cd} |r |||}|||n#|||wxYw|SrS)r)rTextra_exception_cbrrrets r"rHzOauthUrlHelper._exception_cbs` .! 9((i88   c9 - - - -D  c9 - - - - s )Acti}|r ||}||||SrS)rr)rTextra_headers_cbr r;s r"rGzOauthUrlHelper._headers_cbsB  ,&&s++Gts++,,,r$)NNNNr) r`rarbrUr(r:rrrJrrIrHrGr/r$r"rrs- ####6 & & &8   $---999444r$rc ddlm}n"#t$r}td|d}~wwxYw|rt |}nd}||||||j|}||\} } } | S)Nrzoauth support is not available) client_secretresource_owner_keyresource_owner_secretsignature_methodrD)oauthlib.oauth1oauth1 ImportErrorNotImplementedErrorr*ClientSIGNATURE_PLAINTEXTsign) r rr!r"r rDrXrMclient_urisigned_headers_bodys r"rErE&sK((((((( KKK!"BCCJK NN  ]]%$*3 F#)++c"2"2D.% s (#() retry_codesretry_instancesct|tsdS|j|vrdS|jrt|j|rdSdS)aConfigurable retry exception callback for readurl(). :param retry_codes: Codes to retry on. Defaults to 404. :param retry_instances: Exception types to retry on. Defaults to requests.Timeout. :returns: False to raise the exception from readurl(), True to retry. FT)rrHr:r9)rexcrbrcs r"retry_on_url_excrf?sT c8 $ $u x;t yZ ?;;t 5r$)NNrrNNNNTTNNFTrFrS)rr)@rr1r/rrconcurrent.futuresrrr email.utilsrrGr functoolsr http.clientr itertoolsr typingr r r rrrr urllib.parserrrryr cloudinitrloggingr getLoggerr`rArr#r2rNrPr4r5rFrHrr~rIr*r}rrrrrrrrETimeoutrfr/r$r"rrs  MMMMMMMMMM!!!!!!!!!!!!HHHHHHHHHHHHHHHHHH4444444444$$$$$$g!! """    &u-J'K&&&&> - - - - - - - - > .K.K.K.K.K.K.K.Kbw(      #qq" #q$%qqqqr  38   ?     *! P-P- 38 P-CyP-P- P-  P-P-P-P-j )!/3"&NN N  NNNS#JO,NN NNNNNbxxxxxxxxxLP4(\H