|
If I build everything using ninja, one of the generated targets is build/unittests.txt. This file seems to have incorrect formatting, as this is the error I get when running resmoke:
[resmoke] 2020-07-09T09:32:16.992-0400 verbatim resmoke.py invocation: buildscripts/resmoke.py run --suites=unittests -j30
|
Traceback (most recent call last):
|
File "buildscripts/resmoke.py", line 14, in <module>
|
cli.main(sys.argv)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/cli.py", line 17, in main
|
subcommand.execute()
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/run/__init__.py", line 118, in execute
|
self.run_tests()
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/run/__init__.py", line 176, in run_tests
|
suites = self._get_suites()
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/run/__init__.py", line 260, in _get_suites
|
return suitesconfig.get_suites(config.SUITE_FILES, config.TEST_FILES)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/suitesconfig.py", line 99, in get_suites
|
suite = _suite.Suite(suite_filename, suite_config)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/testing/suite.py", line 74, in __init__
|
self.tests, self.excluded = self._get_tests_for_kind(self.test_kind)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/testing/suite.py", line 105, in _get_tests_for_kind
|
return _selector.filter_tests(test_kind, selector_config)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/selector.py", line 728, in filter_tests
|
return selector.select(selector_config)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/selector.py", line 580, in select
|
return _Selector.select(self, selector_config)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/selector.py", line 428, in select
|
test_list = _TestList(self._test_file_explorer, roots, self._tests_are_files)
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/selector.py", line 154, in __init__
|
self._roots = self._expand_files(roots) if tests_are_files else roots
|
File "/DATA1/data/2/mongo/buildscripts/resmokelib/selector.py", line 164, in _expand_files
|
raise ValueError("Unrecognized test file: {}".format(test))
|
ValueError: Unrecognized test file: build/install/bin/base_test\nbuild/install/bin/bson_mutable_test\nbuild/install/bin/bson_util_test\nbuild/install/bin/bson_test\nbuild/install/bin/topology_description_test\nbuild/install/bin/server_description_test\nbuild/install/bin/topology_manager_test\nbuild/install/bin/server_selector_test\nbuild/install/bin/topology_state_machine_test\nbuild/install/bin/client_test\nbuild/install/bin/client_rs_test\nbuild/install/bin/crypto_test\nbuild/install/bin/db_auth_test\nbuild/install/bin/db_bson_test\nbuild/install/bin/db_catalog_util_test\nbuild/install/bin/db_catalog_test\nbuild/install/bin/map_reduce_agg_test\nbuild/install/bin/command_mirroring_test\nbuild/install/bin/db_commands_test\nbuild/install/bin/db_concurrency_test\nbuild/install/bin/cst_test\nbuild/install/bin/db_sbe_test\nbuild/install/bin/db_exec_test\nbuild/install/bin/db_free_mon_test\nbuild/install/bin/db_fts_unicode_test\nbuild/install/bin/db_fts_test\nbuild/install/bin/db_ftdc_test\nbuild/install/bin/db_geo_test\nbuild/install/bin/db_index_test\nbuild/install/bin/db_matcher_test\nbuild/install/bin/db_ops_test\nbuild/install/bin/process_interface_test\nbuild/install/bin/db_pipeline_test\nbuild/install/bin/db_query_collation_test\nbuild/install/bin/query_datetime_test\nbuild/install/bin/db_query_test\nbuild/install/bin/db_repl_test\nbuild/install/bin/db_repl_coordinator_test\nbuild/install/bin/topology_version_observer_test\nbuild/install/bin/db_repl_cloners_test\nbuild/install/bin/db_s_shard_server_test\nbuild/install/bin/db_s_collection_sharding_runtime_test\nbuild/install/bin/db_s_transaction_coordinator_test\nbuild/install/bin/db_s_config_server_test\nbuild/install/bin/db_sorter_test\nbuild/install/bin/db_stats_test\nbuild/install/bin/storage_biggie_test\nbuild/install/bin/storage_ephemeral_for_test_test\nbuild/install/bin/storage_wiredtiger_test\nbuild/install/bin/storage_wiredtiger_record_store_and_index_test\nbuild/install/bin/storage_wiredtiger_prefixed_record_store_and_index_test\nbuild/install/bin/db_storage_test\nbuild/install/bin/db_update_test\nbuild/install/bin/db_views_test\nbuild/install/bin/client_out_of_line_executor_test\nbuild/install/bin/db_unittests\nbuild/install/bin/db_unittests_with_config_server_test_fixture\nbuild/install/bin/mongo_embedded_test\nbuild/install/bin/stitch_support_test\nbuild/install/bin/executor_test\nbuild/install/bin/idl_test\nbuild/install/bin/logger_test\nbuild/install/bin/logv2_test\nbuild/install/bin/platform_test\nbuild/install/bin/rpc_test\nbuild/install/bin/s_catalog_test\nbuild/install/bin/s_commands_test\nbuild/install/bin/s_query_test\nbuild/install/bin/s_test\nbuild/install/bin/scripting_test\nbuild/install/bin/shell_test\nbuild/install/bin/stdx_test\nbuild/install/bin/sigaltstack_location_test\nbuild/install/bin/set_terminate_dispatch_test\nbuild/install/bin/set_terminate_from_main_die_in_thread_test\nbuild/install/bin/set_terminate_from_thread_die_in_main_test\nbuild/install/bin/set_terminate_from_thread_die_in_thread_test\nbuild/install/bin/transport_test\nbuild/install/bin/unittest_test\nbuild/install/bin/util_concurrency_test\nbuild/install/bin/util_net_test\nbuild/install/bin/util_net_ssl_test\nbuild/install/bin/options_parser_test\nbuild/install/bin/thread_safety_context_test\nbuild/install/bin/util_test\nbuild/install/bin/stacktrace_libunwind_test\nbuild/install/bin/stacktrace_test\nbuild/install/bin/watchdog_test
|
|
|
Author:
{'name': 'Daniel Moody', 'email': 'daniel.moody@mongodb.com', 'username': 'dmoody256'}
Message: SERVER-49395 more portable way for ninja to build unittests.txt
(cherry picked from commit ab1a892520e9a9b84ae90d52f84a0f107efbdecd)
Branch: v4.4
https://github.com/mongodb/mongo/commit/1fc20a5ec81db278996ab2a5e0f535b30134b5b9
|
|
Author:
{'name': 'Daniel Moody', 'email': 'daniel.moody@mongodb.com', 'username': 'dmoody256'}
Message: SERVER-49395 more portable way for ninja to build unittests.txt
Branch: master
https://github.com/mongodb/mongo/commit/ab1a892520e9a9b84ae90d52f84a0f107efbdecd
|
|
I am assuming that you have a strange setup on your system, and the default system setup works, otherwise this may not have not been the original solution for matt and be working for me? Anyone else watching can add to the consensus?
|
|
I dont know, I was confused by that. It seems echo in that case is not see -ne as args, because I also have a newline at the end. Also ninja, not python is the one interpreting the string in the correct case. But I am not sure what it's doing with echo that is making echo behave wierd.
echo behaves as expected with those args when I run tests with it just from the command line.
|
|
That's weird. Without the patch, how does it not have "-n " at the front?
|
|
I tried that on my system previously, and it does not work for me. I end up with a build/unittest.txt that has "-ne " at the front.
|
|
Daniel, your patch works for me. But I found this patch also works (except it leaves the last line of the file without a newline):
diff --git a/SConstruct b/SConstruct
|
index 0f47acf568..89389a5420 100644
|
--- a/SConstruct
|
+++ b/SConstruct
|
@@ -3946,7 +3946,7 @@ if get_option('ninja') != 'disabled':
|
description="Compiling test list: $out",
|
command="{prefix}echo {flags} '$files' > '$out'".format(
|
prefix="cmd.exe /c " if env["PLATFORM"] == "win32" else "",
|
- flags="-n" if env["PLATFORM"] != "win32" else "",
|
+ flags="-ne" if env["PLATFORM"] != "win32" else "",
|
),
|
)
|
env.NinjaRegisterFunctionHandler("test_list_builder_action", ninja_test_list_builder)
|
|
|
(The bash echo command will not interpret \n's unless you pass -e, as per its documentation. Therefore, I believe that in the correctly behaving case, python has to be interpreting them before passing the data to the echo command.)
|
|
could not repro with python3.7 and ninja 1.9 in the mix. The attached ninja file looks good, so it must be something with the shell on your system that ninja is executing the commands on or the environment that is preventing \n from being interpreted by echo.
I may have a more portable solution. Could you try the attached patch file on the system you can repro it on?
|
|
Thanks I'll try that python and ninja.
|
|
I am using python 3.7.6 and ninja 1.9.0
|
|
I think it might be something to do with the way the Ninja python transforms the build directive (which is to just run the bash echo command with a string and redirect output into the .txt file). In the Ninja file itself, the text is present with \n's, and for some reason on my system, ninja does not transform them into newline characters prior to presenting them to the echo command.
|
|
I couldn't reproduce this on ubuntu 18.04 with python 3.8 and ninja 1.10 after checking out that commit hash and running these commands:
/usr/bin/python3.8 "buildscripts/scons.py" "-Q" "--variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars" "MONGO_VERSION=4.5.0" "MONGO_GIT_HASH=unknown" "ICECC=icecc" "CCACHE=ccache" "--link-model=dynamic" "--dbg=on" "--ninja" "generate-ninja"
|
ninja -j500 build/unittests.txt
|
/usr/bin/python3.8 buildscripts/resmoke.py run --suites=unittests -j30
|
|
|
I discovered today that find-suites has a similar problem:
/data/m/44/mongo: buildscripts/resmoke.py find-suites jstests/core/txns/create_collection.js
|
Traceback (most recent call last):
|
File "buildscripts/resmoke.py", line 24, in <module>
|
main()
|
File "buildscripts/resmoke.py", line 20, in main
|
subcommand.execute()
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/commands/run.py", line 109, in execute
|
self.find_suites()
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/commands/run.py", line 127, in find_suites
|
suites_by_test = self._find_suites_by_test(suites)
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/commands/run.py", line 141, in _find_suites_by_test
|
test_membership = suitesconfig.create_test_membership_map()
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/suitesconfig.py", line 56, in create_test_membership_map
|
suite = _suite.Suite(suite_name, suite_config)
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/testing/suite.py", line 74, in __init__
|
self.tests, self.excluded = self._get_tests_for_kind(self.test_kind)
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/testing/suite.py", line 105, in _get_tests_for_kind
|
return _selector.filter_tests(test_kind, selector_config)
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/selector.py", line 728, in filter_tests
|
return selector.select(selector_config)
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/selector.py", line 581, in select
|
return _Selector.select(self, selector_config)
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/selector.py", line 429, in select
|
test_list = _TestList(self._test_file_explorer, roots, self._tests_are_files)
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/selector.py", line 155, in __init__
|
self._roots = self._expand_files(roots) if tests_are_files else roots
|
File "/media/DATA2/data/m/44/mongo/buildscripts/resmokelib/selector.py", line 165, in _expand_files
|
raise ValueError("Unrecognized test file: {}".format(test))
|
ValueError: Unrecognized test file: build/debug/mongo/bson/bson_bm\nbuild/debug/mongo/db/concurrency/lock_manager_bm\nbuild/debug/mongo/db/storage/wiredtiger/storage_wiredtiger_begin_transaction_block_bm\nbuild/debug/mongo/db/storage/storage_key_string_bm\nbuild/debug/mongo/db/commands_bm\nbuild/debug/mongo/logv2/logv2_bm\nbuild/debug/mongo/platform/endian_bm\nbuild/debug/mongo/s/chunk_manager_refresh_bm\nbuild/debug/mongo/stdx/condition_variable_bm\nbuild/debug/mongo/unittest/system_resource_canary_bm\nbuild/debug/mongo/util/clock_source_bm\nbuild/debug/mongo/util/decimal_counter_bm\nbuild/debug/mongo/util/itoa_bm\nbuild/debug/mongo/util/future_bm\nbuild/debug/mongo/util/hash_table_bm\nbuild/debug/mongo/util/base64_bm\nbuild/debug/mongo/util/stacktrace_bm
|
(The above text is from build/benchmarks.txt, a file generated in a similar way to unittests.txt)
I wish I could figure out what about my system is preventing translation from \n to literal newlines.
|
|
Sure – I don't have the scons invocation but I think the top of the ninja file gives you basically that. I'll attach it here. The commit was 0e8bb99b4f9576d8e2ab32af4a8ab60efd80a083
build.ninja.zip
|
|
milkie - Can you include the SCons command you used to generate the Ninja file and the commit at which you encountered this? And, if JIRA limits will let you, attach the Ninja file as well.
|
Generated at Thu Feb 08 05:19:44 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.