VanillaPipeline.get_train_loss_dict: 12.6875
Traceback (most recent call last):
File "/home/mare/anaconda3/envs/nerfstudio/bin/ns-train", line 8, in
sys.exit(entrypoint())
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/scripts/train.py", line 262, in entrypoint
main(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/scripts/train.py", line 247, in main
launch(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/scripts/train.py", line 189, in launch
main_func(local_rank=0, world_size=world_size, config=config)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/scripts/train.py", line 100, in train_loop
trainer.train()
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/engine/trainer.py", line 259, in train
loss, loss_dict, metrics_dict = self.train_iteration(step)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/utils/profiler.py", line 127, in inner
out = func(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/engine/trainer.py", line 479, in train_iteration
_, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/utils/profiler.py", line 127, in inner
out = func(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/pipelines/base_pipeline.py", line 298, in get_train_loss_dict
ray_bundle, batch = self.datamanager.next_train(step)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/data/datamanagers/base_datamanager.py", line 542, in next_train
ray_bundle = self.train_ray_generator(ray_indices)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1527, in _call_impl
return forward_call(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/model_components/ray_generators.py", line 56, in forward
ray_bundle = self.cameras.generate_rays(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/cameras/cameras.py", line 463, in generate_rays
raybundle = cameras._generate_rays_from_coords(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/cameras/cameras.py", line 646, in _generate_rays_from_coords
coord_stack[coord_mask, :] = camera_utils.radial_and_tangential_undistort(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/eval_frame.py", line 328, in _fn
return fn(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/eval_frame.py", line 490, in catch_errors
return callback(frame, cache_entry, hooks, frame_state)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 641, in _convert_frame
result = inner_convert(frame, cache_size, hooks, frame_state)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 133, in _fn
return fn(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 389, in _convert_frame_assert
return _compile(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 569, in _compile
guarded_code = compile_inner(code, one_graph, hooks, transform)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 189, in time_wrapper
r = func(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 491, in compile_inner
out_code = transform_code_object(code, transform)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/bytecode_transformation.py", line 1028, in transform_code_object
transformations(instructions, code_options)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 458, in transform
tracer.run()
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2074, in run
super().run()
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 724, in run
and self.step()
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 688, in step
getattr(self, inst.opname)(inst)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2162, in RETURN_VALUE
self.output.compile_subgraph(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 833, in compile_subgraph
self.compile_and_call_fx_graph(tx, list(reversed(stack_values)), root)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/contextlib.py", line 75, in inner
return func(*args, **kwds)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 957, in compile_and_call_fx_graph
compiled_fn = self.call_user_compiler(gm)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 189, in time_wrapper
r = func(*args, **kwargs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1024, in call_user_compiler
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1009, in call_user_compiler
compiled_fn = compiler_fn(gm, self.example_inputs())
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/_dynamo/repro/after_dynamo.py", line 117, in debug_wrapper
compiled_gm = compiler_fn(gm, example_inputs)
File "/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/torch/__init__.py", line 1607, in __call__
return self.compiler_fn(model_, inputs_, **self.kwargs)
torch._dynamo.exc.BackendCompilerFailed: backend='eager' raised:
TypeError: eager() got an unexpected keyword argument 'mode'
Set TORCH_LOGS="+dynamo" and TORCHDYNAMO_VERBOSE=1 for more information
You can suppress this exception and fall back to eager by setting:
import torch._dynamo
torch._dynamo.config.suppress_errors = True
..Solution
* find where train.py is installed
(nerfstudio) mare@mare>NerfStudio$ find /home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/ -name 'train.py'
in my case it installed in here:
/home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/scripts/train.py
(nerfstudio)
* edit train.py code
nano /home/mare/anaconda3/envs/nerfstudio/lib/python3.8/site-packages/nerfstudio/scripts/train.py
add this two line on top of code
import torch._dynamo
torch._dynamo.config.suppress_errors = True
ok it's my solution.
Thank you.
www.marearts.com
No comments:
Post a Comment