^cy|ddlZddlZddlmZddlmZddlmZmZm Z m Z m Z m Z mZmZddlmZddlmZmZmZmZmZmZe jdZGd d ejZd!d Zd"dZd"dZdZ dZ!dZ" d#dZ#dZ$dZ%dZ&dZ' d$dZ(d"dZ)GddZ*GddZ+GddZ,d Z-dS)%N)_short)encodingerrorlockmdiffpolicyscmutilutilvfs)procutil) constants contentstoredatapack historypack metadatastore shallowutilosutilceZdZdS)RepackAlreadyRunningN)__name__ __module__ __qualname__ #$$$<==# >"""GLL F w($<==/ $} #x/MMUMfMMMMMrcptj|drgtj|j}t j|jddi}tj |tj }t||||tj |tj|j drt|\}}|\}}} |\} } } tj| }t j| ddi}t|||| tj|tj|ddi}t j| ddi}t||||tj|dSdS)zIIf ``packsonly`` is True, stores creating only loose objects are skipped.shareddatastoresallowincompleteToptions datastoreN)r safehasattrrunioncontentstorer4runionmetadatastoresharedhistorystoresrgetcachepackpathrFILEPACK_CATEGORY _runrepack manifestlog_getmanifeststoresTREEPACK_CATEGORY) r,r7 datasource historysourcepackpath localdata shareddata lpackpathldstoreslhstores spackpathsdstoresshstoress r fullrepackrN2s 011 !3T5JK %8  % 7;  / )-         '      (+66! 24 8 8 :(1% 8X(2% 8X"3X> %8  '+         '     "3  '+  &8  '+         '       5! ! rc tj|drHtj|tj}t ||j|j|tj|tj|j dr_t|\}}|\}}}|\}} } t || | |tj |t ||||tj d|dSdS)zThis repacks the repo by looking at the distribution of pack files in the repo and performing the most minimal repack to keep the repo in good shape. r4r6r8T)allowincompletedatar7N) r r9rr=rr>_incrementalrepackr4r<r@rArB) r,r7rErFrGrHrIrJrKrLrMs rincrementalrepackrRjs 011  / )-      !  $   '      (+66 24 8 8 :(1% 8X(2% 8X       '            ' $      !  rc|jj}|jj}|jj}|jj}t j|tj}t j |j j j tj}|||f|||ffSN) r@r4localdatastoresr<localhistorystoresrr=rrBgetlocalpackpathsvfsr base)r,r4rUr<rVsharedpackpath localpackpaths rrArAs'8&6O*>)< 1 i)N 0 I7M );< )+>? rcvtfd|D}tfd|D}|S)Nc3XK|]$}tj|V%dSrT)ospathjoin).0prEs r z_topacks..s3::qh**::::::rc3.K|]}|VdSrTr)rarb constructors rrcz_topacks..s+//AQ//////r)list)rEfilesrepathspackss` ` r_topacksrjsM ::::E::: : :E /////// / /E Lrc  |jdddkr|Sddgfd|D D]}tj||}D]c}||z}|jd|t jtj|j fztj |d fd|DS) zsDeletes packfiles that are bigger than ``packs.maxpacksize``. Returns ``files` with the removed files omitted.packss maxpacksizers .datapacks.dataidxch|]]\}}}|jktj|dv8tj|d^S)rr)st_sizer^r_splitext)rar_ftypestat VALIDEXTSmaxsizes r z"_deletebigpacks..sj D% L7 " "(8(8(>(>q(AY(N(N q!(N(N(Nrs#removing oversize packfile %s (%s) cbg|]+}tj|dv)|,Sr)r^r_basename)rarow oversizeds r z#_deletebigpacks..s7 N N NCBG$4$4SV$<$D GMM7 (=>>?@    IdOOOO   O N N N5 N N NNrc  tj|j|t|d}t |||}t |t|j|tj}| d|Dt |t|j|tj } | d|Dt|tj tj} t |d| Dtj } | d|Dt|t!j|d|it%j| ddi||t%j| ddi|dS) NT)rqc3NK|] }t|tj|V!dSrT) isinstancer datapackstorerass rrcz%_incrementalrepack..sHJq(2H$I$I rc3NK|] }t|tj|V!dSrTrrhistorypackstorers rrcz%_incrementalrepack..sI ![9:: rc3"K|] \}}}|V dSrTr)rafmoderqs rrcz%_incrementalrepack..s(00}q$000000rc3NK|] }t|tj|V!dSrTrrs rrcz%_incrementalrepack..sI ![9:: rr5) fullhistoryr7)rmkstickygroupdirr&rlistdirrrj_computeincrementaldatapackrextend_computeincrementalhistorypackr_allpackfileswithsuffix PACKSUFFIX INDEXSUFFIXr?rr:rr;) r,r8 historystorerEcategoryrPr7rg datapacks historypacksallhistoryfilesallhistorypackss rrQrQs (333 NN8$N / /E D(E 2 2E-dgu==x?PI&tw66L . {%{'>O00000O   &  (;   (,MMM!4  .2        rcD|dd|dd|dd|dd|ddd}t|tjtj}t||S)N remotefilelogsdata.gencountlimitsdata.generationssdata.maxrepackpackssdata.repackmaxpacksizesdata.repacksizelimit gencountlimit generationsmaxrepackpacksrepackmaxpacksizerepacksizelimit) configint configlistr{rrrr_computeincrementalpackr&rgopts packfiless rrr s,,'79NOO &68KLL<< 4  !nn 7  NN 5    D( x"H$8I #9d 3 33rcJ|dd|dddg|dd|ddd|ddd }t|tjtj}t||S) Nrshistory.gencountlimitshistory.generationss100MBshistory.maxrepackpacksshistory.repackmaxpacksizes400MBshistory.repacksizelimitr)rrr{rrrrrrs rrr"s,, 6   4xj  << 7  !nn :H  NN 8    D$( {%{'>I #9d 3 33rcg}d|D}|D]T\}}}||s|dt| }||z|vr<||||fU|S)Nch|]\}}}| Srr)rafnrrqs rrtz*_allpackfileswithsuffix..=s...nb$r...r)endswithlenr%) rg packsuffix indexsuffixresultfilesetfilenamerrqprefixs rrr;s F.....G % , ,$  ,,  ,S__,,- [  0 0  vtT*++++ Mrc  ttd|dDd}|dg}tt |D]}|gi |D]W\}}}|j}||dkr| |<t |D](\}} || kr|||n)Xg} t |D]T\}} t |||dkr0| t||d fd nU| d d } | d d } t fd | D} | |d krz| rxt | |dkr_| | | | dz } | |d kr| rt | |dk_| S)a`Given a set of pack files along with the configuration options, this function computes the list of files that should be packed as part of an incremental repack. It tries to strike a balance between keeping incremental repacks cheap (i.e. packing small things when possible, and rolling the packs up to the big ones over time). c3>K|]}tj|VdSrT)r sizetointrs rrcz*_computeincrementalpack..Ws,@@aq!!@@@@@@rrT)reverserrrc|SrTr)xsizess rz)_computeincrementalpack..ws 58r)rkeyNc3(K|] }|V dSrTr)ranrs rrcz*_computeincrementalpack..~s'33!U1X333333rrr) rfsortedr%rangerrn enumeratersumpop)rgrlimits generationsirrrqsizelimitgenpacks chosenpacks repacksizers @rrrLsi@@4+?@@@$OOOF MM!K 3v;;  2 E#  d| $+, , , f !&))  HAue||A%%f---Hf%%5 {1~  &6!7 7 7 OO{1~t9K9K9K9KLLL    E 8233-K}H3333{33333JT,---  .   t$56 6 68<<>>***eKO,, T,---  .   t$56 6 6 rc tj|j|d}|jdd}|s|}t ||||||||} t j|j|5} tj|j|5} | | | n/#tj $rttdwxYw dddn #1swxYwYddddS#1swxYwYdS)Nc|||}||}|jdd}t j|z }|d|kS)zCheck if the file node is older than a limit. Unless a limit is specified in the config the default limit is taken. )fileidrsnodettlr)filectxlinkrevdater&rtime)r,rnoderfiletimettlrs risoldz_runrepack..isoldsp,,x,55))*//11g 0*== c!{U""rrsgcrepack)gcrr7s3skipping repack - another repack is already running)rrr&r(repackerrmutabledatapackrmutablehistorypackrunrLockHeldrr) r,datahistoryrErrr7rgarbagecollectpackerdpackhpacks rr?r?s (333 # # #W''(8+FFN        F  !$'8 4 4   +DGX > > %  5%((((>   *. )                                 sN.C6 C B"!C",CC C6C" "C6%C" &C66C:=C:c |st}n|}gd}|jddd}|r|d|z|jddd}|r|d|zd|}d|zg}t j||}t}d}t|jD]{} ||  |vr-||  } nJ|r.||  |} n||  } ||  }|| t!| t"urU| D]>\} } | d d +||| | d d ?B| D]$\} } ||| | %}|S) a Computes a keepset which is not garbage collected. 'keyfn' is a function that maps filename, node to a unique key. 'lastkeepkeys' is an optional argument and if provided the keepset function updates lastkeepkeys with more keys and returns the result. ).sdraft()sparents(draft())rs pullprefetchNs(%s)sbgprefetchrevs+ssort((%s), "topo")r)setr&configr%r`r revrangereversed_listp1rev manifestctx readdeltamanifestdiffaddtypedictitems)r,keyfn lastkeepkeyskeepkeysrevs prefetchrevskeep processed lastmanifestrmrrfilenodes rkeepsetrsU  55 3 2 2D7>>"2OTJJL, Gl*+++7>>"24EtLLL, Gl*+++ 99T??D !D ( )D  D$ ' 'DILdj ! !88 7::<<     * *Q##%%//11AA 'G$$&&++L99G$$&&Aw''))  a 77d??#$'')) > >$71:)LLxa! '(ggii 8 8"( UU8X667777 8 Orc>eZdZdZ d dZdZdZdZdZd Z dS) rz^Class for orchestrating the repack of data and history information into a new format. FNc ||_||_||_||_t j||_||_||_|jr0|stdt|d|_ ||_ dSdS)Ns*Function 'isold' is not properly specifiedc ||fSrTr)rrs rrz#repacker.__init__..s 1vr) r,rrrrgetunitsunitrr7 ValueErrorrrr) selfr,rrrrrrr7s r__init__zrepacker.__init__s   &&x00      P !NOOO#D*=*=>>DMDJJJ   rct}tjt|jddd5|jd|j||j|j ||j| ||| |||j D]}| | ddddS#1swxYwYdS)Ns repacklockr)desctimeouts prerepackr6) repackledgerlockmodr repacklockvfsr,hookr markledgerr7r repackdata repackhistorysourcescleanup)r targetdata targethistoryledgersources rrz repacker.runsD \ $) $ $m$    ' ' INN< ( ( ( I  > > > L # #FDL # A A A OOFJ / / /   v} 5 5 5!. ' 'v&&&& ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 'sB(C//C36C3cs|Sd fd }ttt|d|jr7|dddDfzt D]>\}}|dkrjjdf||<|dz } | || ddzf||<?fd |D}|z }|S) aqReorderes ``orphans`` into a single chain inside ``nodes`` and ``deltabases``. We often have orphan entries (nodes without a base that aren't referenced by other nodes -- i.e., part of a chain) due to gaps in history. Rather than store them as individual fulltexts, we prefer to insert them as one chain sorted by size. rc~j|}tj|vr|tjS|SrT)rgetmetar METAKEYSIZE)rdefaultmetarrs rgetsizez'repacker._chainorphans..getsize2s;9$$Xt44D$,,I122rT)rrs%s: orphan chain: %s s, c,g|]}t|Srrrs rrzz*repacker._chainorphans..As(C(C(Caq(C(C(Crrcg|]}|v| Srr)rarorphanss rrzz*repacker._chainorphans..Ks#666qQg%5%5%5%5%5rrv)rrf debugflagr|r`rr,nullid) rr&rnodesr* deltabasesr'rrparents ` ` ` r _chainorphanszrepacker._chainorphans&sA L       //vg7DAAABB <  HH)UZZ(C(C7(C(C(CDDEF    !)) G GGAtAvv$(I$4a#8 4   Q$*Jv,>q,AA,E#F 4  6666E666  rc  '|jj}|ddd}i}|jD],}|jr#|||ji|j<-d}| td|j t|}t|D] \} } ||i} t!d| D'g} | tdd t'} t#'D]u\}}|| vr | | | |j| || Q#t($r| |YrwxYw| t!t/|| }t| dkr%|d t| z|t| t!t7'fd |}|jrSg}|D]L}| |f|jvr*||j| |rd | |_7||M|}i}tA}tA}tA''| tdd t|}t#|D]\}}|||!|d}|5|jj"d}}|jj"df||<|#|n|\}}|#|| !|}|rO|\}}}}|r |jj"}||kr4||jj"kr ||dzf||<||jj"kr ||dzf||<|$ddr||z }|%|| |||}t#|D]S\}}||\}}||jj"kr|j&'| |}|\} }!}"}#|#!tPj)}$|!| ks|"|ks|$Z|j&!| |}%|j&!| |}&t|&}$tUj+|%|&} nE|j&!| |} t| }$|j&,| |}#tPj)|#vr|$|#tPj)<|#| ||| |#d | |_-U||dz }||.|dS)Nrls maxchainlenirsrepacking datar totalc3K|]}|VdSrTrrars rrcz&repacker.repackdata..`"22$222222rsbuilding historysnodesknowns%repackdata: %d nodes without history c |vSrTr)rr-s rrz%repacker.repackdata..s DEMrTsprocessing nodesrr schainorphansbysizer)/r,r&rentriesvaluesrC setdefaultrr makeprogressrr rrrupdaterfrr getancestorsKeyErrorr%completer _toposortr|rfilterrrrgcedrgetr,rr(r0rgetdeltarr$r textdiffr# datarepackedclose)(rrtargetr& maxchainlenbyfileentrycountrepackprogressrr; ancestors nohistory buildprogressrr orderednodesneworderednodesr.nobase referencedprocessprogress deltatuple deltabasechainlen ancestorinforp2linknodecopyfromr* deltaentrydelta deltabasename origdeltabaser&r deltabasetextoriginalr-s( @rrzrepacker.repackdataOs Y\ll8^TBB ^**,, J JE JDI!!%."55ejA  tyF )  "( !7!7J J  Hg  ! !% ( ( (I22'22222EIOO%&&XSZZ,M%U++ + +49$$$$Q''' +$$(55$d)6  +++$$T***** +  " " $ $ $  )B)B C CDDL9~~!!=IN   y 1 1 2 2 2 1111<@@L " /"$(11D!$'t}<< 8TBB<.2 * #**40000. JUUFJJJE oo%&&XS=N=N.O%\22" B" B4&&q)))(^^D$77 %*.)*:AxI(, (8!'>  :-#11,   %\22 2 24&0&6# 8  000!%!3!3Hd!C!CJ@J=E=-88I$9::D%11(I55<(, h (J(J #'9==4#@#@"8}} %}h G G IMM(D99Eu::D9,,Xt<.r6rr7Tr:)r,r&r;r<rDr=rrr>rr rrrrfr?rr@rrCrr,rhistoryrepacked incrementrBrJ)rrrKr&rMrNprogressrr;rQr-rrT dontprocessrr]r^r_s rrzrepacker.repackhistorysc Y\^**,, J JE" JDI!!%."55ejA?? " # #$)3v;;#  "( !7!7, !, ! HgI22'22222E  9$$  L--hI-NN $DNN9$=$=>>L%%K$ 9 9-6t_*B(;&&4w+>+>TY---(h2F2F#+++TY---#+++(OOB''' 8T2r8XFFF7??48GDM1     F #####rcdfd}tj|}|S)Nc|\}}}}g}|jjkr|||jjkr|||SrT)r,r,r%)rrr]r^r_parentsrQrs r parentfuncz&repacker._toposort..parentfunc(sc)24 &BHhGTY%%%r"""TY%%%r"""Nr)r sortnodeskeys)rrQro sortednodess`` rrCzrepacker._toposort'sF      "+INN,<,r rX)r,sharedcachepaths rrrsM f%%&6 )-  w'''yr)TFrT)FN)NN).r^rmercurial.i18nrmercurial.noder mercurialrrrrr r r r r mercurial.utilsrrrrrrr importmodrAbortrr2rNrRrArjrrQrrrrr?rrrr~rrrrrs                      %$$$$$  ( # #     5;   NNNN"5 5 5 5 p) ) ) ) X& OOOL ;;;;|444*4442";;;~HL))))X;;;;|~~~~~~~~B - - - - - - - - `8     r