^cXddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z m Z m Z mZmZmZddlmZmZddlmZmZmZejZdadadadadZdZd Z d Z!Gd d Z"d Z#dZ$dZ%GddZ&dS)N)_)binhex)errorpycompatrevlogsshpeerutilwireprotov1peer)hashutilprocutil) constants contentstore metadatastorecttj|}tj||dd|dd|S)Nrr sha1digestospathjoin)reponamefileidpathhashs F/usr/lib/python3/dist-packages/hgext/remotefilelog/fileserverclient.py getcachekeyr/sO8=&&--//00H 7<<(2A2, b A AActtj|}tj||SNr)rrrs r getlocalkeyr#4s;8=&&--//00H 7<<" % %%r c@Gfdd|j|_dS)NcteZdZejdZejdZdZfdZfdZ xZ S)!peersetup..remotefilepeercr|dstjdfd}d|fS)N x_rfl_getfiles5configured remotefile server does not support getfilec|dd\}}t|rtj||S)Nr)splitintr LookupError)dcodedatarnodes rdecodez?peersetup..remotefilepeer.x_rfl_getfile..decodeBsBWWUA.. dt99>+D$=== r sfilesnodecapablerAbort)selfrr1r2s `` r x_rfl_getfilez/peersetup..remotefilepeer.x_rfl_getfile;sc<< 011 kL       "D1169 9r cf|dstjdd}d|i|fS)Nsx_rfl_getflogheadss:configured remotefile server does not support getflogheadsc4|r|dngS)N )r+)r.s rr2zDpeersetup..remotefilepeer.x_rfl_getflogheads..decodeRs)*2qwwu~~~2r spathr4)r7rr2s rx_rfl_getflogheadsz4peersetup..remotefilepeer.x_rfl_getflogheadsJsP<< 566 k,  3 3 3T?F* *r cx|dkrdStj|vrdStj|dsdStj|jjvrdS|d}|r|g}ng}| tj |jj r9d |jj }d|z}| ||jj r9d |jj }d|z}| |d ||d<dS)Ns getbundle _localrepos bundlecapsr*sincludepattern=sexcludepattern=,)rNETWORK_CAP_LEGACY_SSH_GETFILES capabilitiesr safehasattrSHALLOWREPO_REQUIREMENTr> requirementsgetappendBUNDLE2_CAPABLITYincludepatternrexcludepattern)r7commandopts bundlecapspatterns includecap excludecaps r_updatecallstreamoptsz7peersetup..remotefilepeer._updatecallstreamoptsWsO,&&9((**++#D,77 1344-00J (\     i9 : : :- . ::do&DEE/(: !!*---- . ::do&DEE/(: !!*---"&))J"7"7D   r cj|||t|j||fi|Sr")rPsuper _sendrequest)r7rJargsrK __class__remotefilepeers rrSz.peersetup..remotefilepeer._sendrequestsL  & &w 5 5 5;5..;!% r ct|}tj|ds(||t j|t|j|fi|S)NrS)rRr rBrPr byteskwargs _callstream)r7rJrK supertyperUrVs rrYz-peersetup..remotefilepeer._callstreamsnnd33I#I~>> P**7H4H4N4NOOO:5..:7KKdKK Kr ) __name__ __module__ __qualname__r batchabler8r<rPrSrY __classcell__)rUrVs@rrVr&:s  " : : # " :  " + + # " +& 8& 8& 8P        L L L L L L L L L Lr rV)rU)uipeerrVs @r peersetuprb9s[OLOLOLOLOLOLOLOLOLOLb$DNNNr c2eZdZdZdZdZdZd dZdZdS) cacheconnectionzThe connection for communicating with the remote cache. Performs gets and sets by communicating with an external process that has the cache-specific implementation. cLdx|_x|_|_d|_d|_dS)NF)pipeopipeipipee subprocess connectedr7s r__init__zcacheconnection.__init__s)/33 3TZ$*r c|jr!tjtdt j|\|_|_|_|_d|_ dS)Nscache connection already openT) rfrr6rr popen4rgrhrirj)r7 cachecommands rconnectzcacheconnection.connectsX : C+a @AABB B>Fo ? ? ; DJ DOr cnd}|jr |jdn#t$rYnwxYw||jd|_||jd|_||jd|_ |jn#t$rYnwxYwd|_d|_dS)NcR |dS#t$rYdSwxYwr")close Exception)pipes rtryclosez'cacheconnection.close..trycloses:      s  &&sexit F)rjrgwritertrfrhri communicate)r7rvs rrszcacheconnection.closes    > #    ++++     HTZ DJ HTZ DJ HTZ DJ ++----    "DOs' 44=B B$#B$Tc|jr_ |j||r|jdSdS#t$r|YdSwxYwdSr")rjrgrwflushIOErrorrs)r7requestrzs rr|zcacheconnection.requests >     )))'J$$&&&&&''        s5AA$#A$c|jsdS |jdd}|s|n$#t$r|YnwxYw|S)N)rjrfreadlinersr{)r7results r receivelinezcacheconnection.receivelines{~ 4 Z((**3B3/F      JJLLLLL  s7AA$#A$NT) r[r\r]__doc__rlrprsr|rr rrdrdsn  8     r rdc |5}g}|D]<}||d|||ddd=t|D]q\} }|| } d|| <||} |dd} |t jdt| | fz| | |r ddddS#1swxYwYdS)Nr(r3s%d %s)commandexecutorrF callcommand enumeraterioBytesIOlen) remotereceivemissing progresstickmissedidmap batchsizeefuturesmirfile_r1s r_getfilesbatchrs`    ! !Q  A NN $a1STT7&K&K     f%%  DAq !!##AGAJ!HESTT7D N2:i3q661+&=>>t L L L LNNNN  sCC&&C*-C*c@|dd}|j}|j}|t|kr|} t t|| |z} | }|| | D]0} | dd} || } d| | fz}||1||| | D]+} | dd} || } ||| | |,|t|k|d|dS)Nx_rfl_getfilesrr%s%s r;)rY_pipeo_pipeirminrwrz)rrrrrsteprrfrgstartend missingid versionidr sshrequests r_getfiles_optimisticrsF ())) A ME ME c&kk//#f++ut|,, c * $ $I!#$$I#D"i%66J KK # # # #  c *  I!#$$I#D N5$ 2 2 2 LNNNN% c&kk//* KK KKMMMMMr c |d|j |j} fd}tj|}d|_|D]+} | dd} | } ||| | |,| d dS)NrcD]0}|dd}|}d||fz}|1dS)Nrr)rwrz)rrrrrrrfs rwriterz"_getfiles_threaded..writers^ $ $I!#$$I#D"i%66J KK # # # # r )targetTrr;) rYrr threadingThreaddaemonrrrwrz) rrrrrrrgr writerthreadrrrrfs `` @r_getfiles_threadedrs ())) ME ME#6222LL cddO YudI...  KK KKMMMMMr cReZdZdZdZdZdZdZdZdZ dZ dd Z d Z d Z dS)fileserverclientz:A client for requesting files from the remote file server.cB|j}||_||_|dd|_|jrt j|j|_|dd|_|dd|_t|_ dS)N remotefilelogs cacheprocessscacheprocess.includepathsdebug) r`repoconfig cacheprocessr expandpath configboolcacheprocesspasspath debugoutputrd remotecache)r7rr`s rrlzfileserverclient.__init__4s W II&6HH   C $0A B BD %'MM 9% % !==)98DD*,,r c>||_||_||_||_dSr") datastore historystore writedata writehistory)r7rrrrs rsetstorezfileserverclient.setstoreFs&"("(r cT|jj|jjSr")rconnectionpoolrE fallbackpathrks r_connectzfileserverclient._connectLs y'++DI,BCCCr c  |jjs||j}|j}|j}t |}d|z}i}|j}|D]2\} } t|| | } |jr|| dzz }|| dzz }| || <3| ||j td|} | dg} |}|sUt| }|D]}||vr| ||j tdnp|d krni|d r>|d }| t+|d | |t,t | z a|t | z }| |||j d d||||t1jd } | r_|j j}d|j _ |5}|j}|t<jrtA|tBstEj#d|j $d d}|j %d d}|dkrtL}ntN}|||j(| j| ||n|dr|drd}nd}|j $d d|}|j )dt | ztU||j(| j| ||ntEj#ddddn #1swxYwY|j dd| j+|z ||z n8#tX$r+|j dd | j+|z ||z wxYw ||j _n#||j _wxYwd!t | d-| fz}| || .|/|jj0t1j|dS#t1j|wxYw)"zTakes a list of filename/node pairs and fetches them from the server. Files are stored in the local cache. A list of nodes that the server couldn't find is returned. If the connection fails, an exception is raised. sget %d r*r;s downloading)totalrTs@warning: cache connection closed early - falling back to server 0s_hits__rrs"remote cache hit rate is %r of %r )hitrFs"remotefilelog requires ssh serverss getfilessteps getfilestypesthreadedr(sbatchd s batchsizes1requesting %d files from remotefilelog server... s>configured remotefilelog server does not support remotefilelogNsremotefilefetchlogsSuccess ) fetched_filestotal_to_fetchsFail s set %d %s )1rrjrprrrnamerrr|r` makeprogressrupdatersetrFwarn startswithr+ incrementr, fetchmisseslogrumaskverboserrar5rr@ isinstance _sshv1peerrr6 configintrrrrdebugrposrtrcompletemarkrepor)r7fileidscacherrrr|rrrrfullidprogressrr missedsetparts fromcacheoldumaskrconnrr getfilestype _getfiles batchdefaultrs rr|zfileserverclient.requestOs )  LLNNN N yG &9 ! !HD" 444F( (4%<' v~ %G E&MM g7''.(9(9'GG %))++I KK !&11I$ 11 i000 6 D  ##I.. !--""3uQx==111 MM) $ $ $- %2 s6{{" CKK'  ///   2      8E??W N ''/"'E.3D!%!>>%E1$.fj#A#A"&+k$I'"'"!"$(7#4#4 0/$$D,07>> 0/,,L ,{::,> ,@ %I & $ 3 ( 2 & % $ $^^,<==%~~h772/2 /1 (,(9(9 0, ))I!GMM!=?B6{{!K+ & $ 3 ( 2 & % ) #(+!C##a333333333333333jGKK-$&.lY&>',y'8   !GKK-!&.lY&>',y'8    '.DGOOgDGO----*c&kk5::f;M;M-NN g&&&         ty~ . . . HX     BHX    s\.S O=EO< O=O  O=O ,O=<Q=5P22Q6 SQA2SS/c|dd}|s/tjtdtdt |}||}t ||krAtjtdtdt ||fz|j|t|tj |dS)Nr~s error downloading file contents:sconnection closed earlysonly received %s of %s bytes) rr ResponseErrorrr,readrraddremotefilelognoderzlib decompress)r7rufilenamer1linesizer0s rrzfileserverclient.receivemissings}}ss# %566,-- 4yyyy t99  %566122c$ii5FF  ++ c$ii!6!6     r c|jr2d|j|jjfz}|j|dSGdd}||_dS)Ns%s %sc(eZdZdZdZddZdZdS)-fileserverclient.connect..simplecachec"g|_d|_dS)NT) missingidsrjrks rrlz6fileserverclient.connect..simplecache.__init__ s&(DO%)DNNNr cdSr"rrks rrsz3fileserverclient.connect..simplecache.close sDr Tc|d}|ddkrdS|dd|_|jddS)Nr;rsgetrr~r)r+rrF)r7valuerzliness rr|z5fileserverclient.connect..simplecache.requestsR!KK..EQx6))&+AbDkDOO**400000r cjt|jdkr|jdSdS)Nr)rrpoprks rrz9fileserverclient.connect..simplecache.receivelines24?++a//#2215554r Nr)r[r\r]rlrsr|rrr r simplecachersU***1111     r r)rr_pathrrp)r7cmdrs rrpzfileserverclient.connects   -d/1EFFC   $ $S ) ) ) ) )         ( +{}}D   r c trdttttttz ttz dztfz}|jr|j||jd| ddttttdz|j j r|j dSdS)NsN%d files fetched over %d fetches - (%d misses, %0.2f%% hit ratio) over %0.2fs gY@sremotefilelog.prefetch%s%%i)remotefilelogfetchedremotefilelogfetchesremotefilelogfetchmissesremotefilelogfetchtime) fetchesfetchedrfloat fetchcostrr`rrreplacerrjrs)r7msgs rrszfileserverclient.closes  Bg +,,uW~~=E  C " S!!! GKK) D%((%,%,)4'04'7       % %   " " $ $ $ $ $ % %r FTc j}g}|D]X\}}|dks(t|dks||s4||t |fYj} j} |r&tj|j } tj |j } t} |r(| | ||r(| | |tfd| D} | r-fd| D} |jd| zd| r9t$dz at$d krdt$d krYt$d kr:jd d } | rj| d zdt-| D} t.t| z at1j}| } | r1t5jt9dt| zt:t1j|z z a|dSdS)z.downloads the given file versions to the caches.hgtags*c:g|]\}}|jjkdSr"rnullid).0unusedrr7s r z-fileserverclient.prefetch..]s, I I Ijfb" 8H2H2HT2H2H2Hr c>g|]\}}|jjk||fSrr)rfrr7s rrz-fileserverclient.prefetch..`s7!Ar29I3I3IB3I3I3Ir sDremotefilelog not fetching %d null revs - this is likely hiding bugssremotefilelog-ext)rrrs fetchwarningr;c6g|]\}}|t|fSr)r)rrrs rrz-fileserverclient.prefetch..xs&MMMhdB4R/MMMr sunable to download %d filesN)rr shallowmatchrFrrrrunioncontentstoreshareddatastoresrunionmetadatastoresharedhistorystoresrr getmissingr` develwarnr rr logstacktracesortedr timer|rr6rr  _lfsprefetch)r7rforce fetchdata fetchhistoryr idstocheckrrrrrnullids fetchwarningrs` rprefetchzfileserverclient.prefetch8sy  / /HD"  ""r77b==((..!   tSWWo . . . .N (  $68MNI(;)LUU  @   i22:>> ? ? ?  C   l55jAA B B B I I I IJ I I I    %/J G  57>?,      ' qLG"}}2b==#'7>>(/$$L$; \E%9:::""$$$MM&:L:LMMMJ s: &GIKKEj11J k455JG u, ,I   g & & & & &/ ' 'r c trtj|jjdsdStj|jsdSg}|jjj |D]\}}t|}|j |}| |tj zrr| |}tj|}|} | s||t'|dkrB|jjj| t- fd|DsJdSdS)Nslfslocalblobstorerc3fK|]+}|V,dSr")hasoid)rpstores r z0fileserverclient._lfsprefetch..s7<>$''Dzz$&"99 '||D))O//55eeggyy~~'OOA&&& x==1   IN - 7 7% H H H<<<<8<<<<< < < <   < K|]}tj|VdSr")rsysbytes)rss rr5z1fileserverclient.logstacktrace..s-LLaX&q))LLLLLLr ) tracebackr`rr format_stack)r7rJs rr$zfileserverclient.logstacktraces`   3 HHLL93I3I3K3KLLL L L     r N)FTF)r[r\r]rrlrrr|rrprsr.r'r$rr rrr1sDD---$))) DDD[[[z   &--->%%%6BGI'I'I'I'V===,     r r)'rrrr&rmercurial.i18nrmercurial.noderr mercurialrrrr r r mercurial.utilsr r rrr sshv1peerrr r r rr6rr#rbrdrrrrrr rrRs  ########        BBB &&& R$R$R$jBBBBBBBBJ>@>o o o o o o o o o o r