a ==ic@ @sddlmZddlmZmZddlmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlmZddlmmZddlmZddlZddlmZddlmmZddlZddl Z ddl!Z!ddl"m#Z#e$e e$e$fdd d Z%e eej&e e$e fd d d Z'ej&e e$e fejj(dddZ)d0ejj(ejj(dddZ*ej(ej(dddZ+ej(eej&eej&eej&dddZ,ej-ej.ej/ej0ej1ej2ej3ej4ej5ej6ej4ej7ej8g Z9ej:ej;gZej/ddiZ?eej&e e$ej(fdddZ@eej&e e$ej(fe ej(ej(fdd d!ZAGd"d#d#ZBd1d&d'ZCeBeDd(d)d*ZEGd+d,d,ZFdejGfejj(e e e$e feejGejj(d-d.d/ZHdS)2N)ArgumentTarget)fuse_conv_bn_eval)TypeDictAnyTupleIterableOptionalListcast) ShapeProp) defaultdict)Enum)targetreturncCs&|dd^}}|r|dnd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnamers/home/droni/.local/share/virtualenvs/DPS-5Je3_V2c/lib/python3.9/site-packages/torch/fx/experimental/optimization.py _parent_namesr)patternnodemodulescCst|jdkrdS|jd|f}t||D]d\}}t|tjsDdS|jdkrTdSt|jtsfdS|j|vrvdSt ||j|ur*dSq*dS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)rrrnodes expected_typeZ current_noderrrmatches_module_patterns    r*)rr new_modulecCs<t|jtsJt|j\}}|||j<t||||dSN)r"rr&rsetattr)rrr+ parent_namerrrrreplace_node_module,s r/F)modelrc Cstjtjftjtjftjtjfg}|s0t|}t |}t | }t|j }|D]}|jD]~}t|||r`t|jdjdkrq`||jdj}||j} | jsq`t|| } t|jd|| ||jd||q`qVt ||S)z Fuses convolution/BN layers for inference purposes. Will deepcopy your model by default, but can modify the model inplace as well. rr)nnZConv1dZ BatchNorm1dConv2d BatchNorm2dZConv3dZ BatchNorm3dcopydeepcopyr#symbolic_tracedict named_modulesgraphr(r*rr usersrZtrack_running_statsrr/replace_all_uses_with erase_node GraphModule) r0Zinplacepatternsfx_modelr new_graphrrconvbnZ fused_convrrrfuse2s.           rCcCs*t|}Gdddtjj}||S)z5 Removes all dropout layers from the module. cs8eZdZeeedfeeefedfdd Z Z S)z&remove_dropout..DropoutRemover.)rr kwargsrcs>t|j|tjr*t|dks"J|dSt|||SdS)Nrr)r"Z submodulesr1ZDropoutrsuperr)selfrr rD __class__rrrVsz2remove_dropout..DropoutRemover.call_module) __name__ __module__ __qualname__rrrrr&rr __classcell__rrrGrDropoutRemoverUsrM)r#r6torchZ TransformerZ transform)r0r?rMrrrremove_dropoutOs rO) orig_moduler(inputsoutputscs|t}i|D]}||j}||<q|D] }||fdd}||<q.|fdd|D|t||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. cs|Sr,r)xenvrrhz"extract_subgraph..csg|] }|qSrr).0outputrTrr jrWz$extract_subgraph..)r#Graph placeholderrZ node_copyrYlintr=)rPr(rQrRr@inputnew_noderrrTrextract_subgraph^s   r`cCs t|Sr,) th_mkldnnZMkldnnBatchNorm)a_rrrrV{rWrV)r(rcCsi}|D]r}|jdkrt|jts&J||j}t|tvrtt||tj}t|tj s`Jt |||<t |||q|S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r) r%r"rr&r' mkldnn_maprNfloatr1Moduler4r5r/)r(r old_modulesr cur_moduler+rrrmodules_to_mkldnns   ri)r(rrgcCsJ|D]@}|jdkrt|jts"J||j}||vrt||||qdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. rN)r%r"rr&r/)r(rrgrrhrrr reset_moduless   rjc@seZdZejdddZdS) MklSubgraphfx_graphcCs||_g|_g|_g|_dSr,)rmr( start_nodes end_nodes)rFrmrrr__init__szMklSubgraph.__init__N)rIrJrKr#r[rprrrrrksrk rcs(ddttdfdd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) Nr9rcs|j}dur,|jj|jjtdd|Dtttj dd|j D}t |j ||fdd}|fdd}t jj t|fdd}||kS) NcSsg|]}t|jqSr)rNZrandnshaperXrrrrrZrWz@gen_mkl_autotuner..use_mkl_heuristic..cSsg|]}|jdqS)r)r rtrrrrZrWcs<tD] }|qt}tD] }|}q$t|Sr,)rangetime)frcbeginout)iterswarmuprr benchmarks   z?gen_mkl_autotuner..use_mkl_heuristic..benchmarkcsddddDDS)NcSsg|] }|qSrto_denserXirrrrZrWzRgen_mkl_autotuner..use_mkl_heuristic....cSsg|] }|qSr) to_mkldnnrrrrrZrWrrZ sample_inputs submodulerrrVrWz>gen_mkl_autotuner..use_mkl_heuristic..csSr,rrrrrrVrW)rnrmZ owning_modulergr propagater r r#r$ror`r(rjr9r7r8)r9Z input_nodesZ output_argsr|Zmkl_timeZ no_mkl_timeexample_inputsr?rzrgr{rruse_mkl_heuristicsz,gen_mkl_autotuner..use_mkl_heuristic)rkbool)rrzr{rrrrgen_mkl_autotuners rrrcCst|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )rr()r9rrruse_mkl_lengthsrc@sBeZdZddZedddZeedddZeed d d Zd S) UnionFindcCsdg||_dg||_dS)Nrrsize)rFnrrrrps zUnionFind.__init__)vcCs||j|<d|j|<dS)Nrr)rFrrrrmake_sets zUnionFind.make_set)rrcCsB|j|}||kr|S|dus"J|||j|<tt|j|Sr,)rfindr int)rFrparrrrrs   zUnionFind.find)rbbcCsf||||}}||kr"|S|j||j|kr@||}}||j|<|j||j|7<dSr,)rrr)rFrbrrrrjoins  zUnionFind.joinN)rIrJrKrprrrrrrrrrsr)r0 pass_configtracerrc sVdddtid}|duri}|||dr6t|}|drFt|}|ddurV|St|dtsltd d|dvrtd |dd}|}|t |t |j }t| }Gd d d t}tjD]} |j} | jd krV|| j} t| tvr|j} t| d} | dur| jtjks:Jd| jtdksJdn2| jdkr| jtvrv|j} n| jtvr|j} | |jkr| |jkrtdd| j Dsqֈ!| &t "| j fdd} Wdn1s0Yt#t$t j%j&| | _ '| 2(dd| f}| )|| f|_ Wdq1sP0Yqt*tj|}|_+jD]} | jdkrx| jdkrx| j d}t| j,}|D]2}|jdkr|jdkr|)|-|qt.| j,dkrx-| qxt.j}t/|fddt0jD]\}} | jdkrX| jdkrX|| _12|n| jdkr| jdkr| j ddusJ| j d| _3ntfdd| j4D}t.|dkrq&tdd|DrJt5|}|d| _6|ddD]}7|d|qq&t8fd d}jD]r} t9| d!rP|:| j6j;| t9| d"rt|:| j1j<;| t9| d#r(|:| j3j=;| q(|>D]P}||s|j<|j=D]$} | j d}| )|-| qt?|j||qd}jD]&} | jdks| jdkr|d7}qt@AtBCd$|Dt |}|S)%a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicity run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fuserOmkldnn_layout_optimizeNrrOrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configc@seZdZdZdZdZdS)z*optimize_for_inference..MklSupportrrN)rIrJrKNOYESUNKNOWNrrrr MklSupportsrrz)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functioncSsg|]}|jdkqSr})r)rXargrrrrZ3rWz*optimize_for_inference..csd|fS)Nr) call_methodrrlrrrV6rWz(optimize_for_inference..rr~rrcs0t|dr|jSt|dr,|jSdS)Ncolor start_color)hasattrrrrr)ufrr get_colorSs     z)optimize_for_inference..get_colorcs,g|]$}t|tjr|dur|qSr,)r"r#r$r)rrrrZnrWcss|]}|duVqdSr,rrrrr rrWz)optimize_for_inference..rcstSr,)rkrrlrrrVyrWrr end_colorzmkldnn conversions: )ErupdaterCrOr"r7 RuntimeErrortracer4r5r#r=rootr8rlistr(rr%rr'mkldnn_supportedrnext parametersZdtyperNreZdevicemkldnn_supported_unknownranyr Zinserting_beforeZmap_argr rrrZinserting_afterZ create_noder;rirgr:r<rr enumeraterrrZall_input_nodessortedrrrrrappendrnrovaluesrjlogging getLoggerrIinfor])r0rrZdefault_pass_configrZ cur_tracerr?rrrZsupports_mkldnnrhZsample_parameterZ mkldnn_argsZdense_xrgZprv_noder:userZ num_nodesZcur_idxZ cur_colorsZ other_colorZ mkldnn_graphsr9ZprvZmkldnn_conversionsresultr)rmrrroptimize_for_inferences               6  *                   r)F)rqr)IZtorch.fxr#Z torch.fx.noderrZtorch.nn.utils.fusionrtypingrrrrr r r r rNZtorch.nnr1Ztorch.nn.functionalZ functionalFZtorch.fx.passes.shape_propr r4 collectionsrZtorch.utils.mkldnnutilsZmkldnnraoperatorrvrenumrr&rr$r*rfr/rCrOr`r2ZLinearr3ZReLUZ MaxPool2dZ AvgPool2dZAdaptiveAvgPool2dZreluZ transposeZsigmoidZ avg_pool2dZadaptive_avg_pool2draddmulrZ MkldnnConv2dZ MkldnnLinearrdrirjrkrrrrZTracerrrrrrsV  (      (     .  &