Skip to content

Commit 24ff385

Browse files
A-006A-006
authored
Refactor: Remove dependency on shell commands for file operations (deepmodeling#6826)
* remove dependency on shell * fix compile error * fix gtest error * fix system call in the gtest
1 parent b07057f commit 24ff385

File tree

9 files changed

+102
-44
lines changed

9 files changed

+102
-44
lines changed

‎source/source_base/global_file.cpp‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
#include <mpi.h>
1010
#endif
1111
#include <unistd.h>
12+
#include <sys/stat.h>
13+
#include <sys/types.h>
14+
#include <cerrno>
1215
#include <sstream>
1316
#include "global_function.h"
1417
#include "source_io/module_parameter/parameter.h"
@@ -40,14 +43,15 @@ void ModuleBase::Global_File::make_dir_out(
4043
#endif
4144
int make_dir = 0;
4245
// mohan update 2011-05-03
43-
std::string command0 = "test -d " + PARAM.globalv.global_out_dir + " || mkdir " + PARAM.globalv.global_out_dir;
46+
//std::string command0 = "test -d " + PARAM.globalv.global_out_dir + " || mkdir " + PARAM.globalv.global_out_dir;
4447

4548
int times = 0;
4649
while(times<GlobalV::NPROC)
4750
{
4851
if(rank==times)
4952
{
50-
if ( system( command0.c_str() ) == 0 )
53+
int ret = mkdir(PARAM.globalv.global_out_dir.c_str(), 0755);
54+
if ( ret == 0 || errno == EEXIST )
5155
{
5256
std::cout << " MAKE THE DIR : " << PARAM.globalv.global_out_dir << std::endl;
5357
make_dir = 1;
@@ -78,14 +82,15 @@ void ModuleBase::Global_File::make_dir_out(
7882
if(calculation == "md")
7983
{
8084
int make_dir_stru = 0;
81-
std::string command1 = "test -d " + PARAM.globalv.global_stru_dir + " || mkdir " + PARAM.globalv.global_stru_dir;
85+
//std::string command1 = "test -d " + PARAM.globalv.global_stru_dir + " || mkdir " + PARAM.globalv.global_stru_dir;
8286

8387
times = 0;
8488
while(times<GlobalV::NPROC)
8589
{
8690
if(rank==times)
8791
{
88-
if ( system( command1.c_str() ) == 0 )
92+
int ret = mkdir(PARAM.globalv.global_stru_dir.c_str(), 0755);
93+
if ( ret == 0 || errno == EEXIST )
8994
{
9095
std::cout << " MAKE THE STRU DIR : " << PARAM.globalv.global_stru_dir << std::endl;
9196
make_dir_stru = 1;
@@ -118,16 +123,17 @@ void ModuleBase::Global_File::make_dir_out(
118123
if((out_dir) && calculation == "md")
119124
{
120125
int make_dir_matrix = 0;
121-
std::string command1 = "test -d " + PARAM.globalv.global_matrix_dir + " || mkdir " + PARAM.globalv.global_matrix_dir;
126+
//std::string command1 = "test -d " + PARAM.globalv.global_matrix_dir + " || mkdir " + PARAM.globalv.global_matrix_dir;
122127

123128
times = 0;
124129
while(times<GlobalV::NPROC)
125130
{
126131
if(rank==times)
127132
{
128-
if ( system( command1.c_str() ) == 0 )
133+
int ret = mkdir(PARAM.globalv.global_matrix_dir.c_str(), 0755);
134+
if ( ret == 0 || errno == EEXIST )
129135
{
130-
std::cout << " MAKE THE MATRIX DIR : " << PARAM.globalv.global_stru_dir << std::endl;
136+
std::cout << " MAKE THE MATRIX DIR : " << PARAM.globalv.global_matrix_dir << std::endl;
131137
make_dir_matrix = 1;
132138
}
133139
else
@@ -157,14 +163,15 @@ void ModuleBase::Global_File::make_dir_out(
157163
if(out_wfc_dir)
158164
{
159165
int make_dir_wfc = 0;
160-
std::string command1 = "test -d " + PARAM.globalv.global_wfc_dir + " || mkdir " + PARAM.globalv.global_wfc_dir;
166+
//std::string command1 = "test -d " + PARAM.globalv.global_wfc_dir + " || mkdir " + PARAM.globalv.global_wfc_dir;
161167

162168
times = 0;
163169
while(times<GlobalV::NPROC)
164170
{
165171
if(rank==times)
166172
{
167-
if ( system( command1.c_str() ) == 0 )
173+
int ret = mkdir(PARAM.globalv.global_wfc_dir.c_str(), 0755);
174+
if ( ret == 0 || errno == EEXIST )
168175
{
169176
std::cout << " MAKE THE WFC DIR : " << PARAM.globalv.global_wfc_dir << std::endl;
170177
make_dir_wfc = 1;
@@ -196,14 +203,15 @@ void ModuleBase::Global_File::make_dir_out(
196203
if(PARAM.inp.of_ml_gene_data == 1)
197204
{
198205
int make_dir_descrip = 0;
199-
std::string command1 = "test -d " + PARAM.globalv.global_mlkedf_descriptor_dir + " || mkdir " + PARAM.globalv.global_mlkedf_descriptor_dir;
206+
//std::string command1 = "test -d " + PARAM.globalv.global_mlkedf_descriptor_dir + " || mkdir " + PARAM.globalv.global_mlkedf_descriptor_dir;
200207

201208
times = 0;
202209
while(times<GlobalV::NPROC)
203210
{
204211
if(rank==times)
205212
{
206-
if ( system( command1.c_str() ) == 0 )
213+
int ret = mkdir(PARAM.globalv.global_mlkedf_descriptor_dir.c_str(), 0755);
214+
if ( ret == 0 || errno == EEXIST )
207215
{
208216
std::cout << " MAKE THE MLKEDF DESCRIPTOR DIR : " << PARAM.globalv.global_mlkedf_descriptor_dir << std::endl;
209217
make_dir_descrip = 1;
@@ -237,14 +245,15 @@ void ModuleBase::Global_File::make_dir_out(
237245
if(PARAM.inp.deepks_out_freq_elec > 0)
238246
{
239247
int make_dir_deepks_elec = 0;
240-
std::string command1 = "test -d " + PARAM.globalv.global_deepks_label_elec_dir + " || mkdir " + PARAM.globalv.global_deepks_label_elec_dir;
248+
//std::string command1 = "test -d " + PARAM.globalv.global_deepks_label_elec_dir + " || mkdir " + PARAM.globalv.global_deepks_label_elec_dir;
241249

242250
times = 0;
243251
while(times<GlobalV::NPROC)
244252
{
245253
if(rank==times)
246254
{
247-
if ( system( command1.c_str() ) == 0 )
255+
int ret = mkdir(PARAM.globalv.global_deepks_label_elec_dir.c_str(), 0755);
256+
if ( ret == 0 || errno == EEXIST )
248257
{
249258
std::cout << " MAKE THE DEEPKS LABELS (ELEC) DIR : " << PARAM.globalv.global_deepks_label_elec_dir << std::endl;
250259
make_dir_deepks_elec = 1;

‎source/source_base/global_function.cpp‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@
1616
#include <fstream>
1717
#include <iostream>
1818
#include <string>
19+
#include <sys/stat.h>
20+
#include <sys/types.h>
21+
#include <cerrno>
22+
#include <unistd.h>
23+
1924
namespace ModuleBase
2025
{
2126
namespace GlobalFunc
@@ -61,20 +66,14 @@ void OUT(std::ofstream &ofs, const std::string &name)
6166
void MAKE_DIR(const std::string &fn)
6267
{
6368
// ModuleBase::TITLE("global_function","MAKE_DIR");
64-
#ifndef __SW
6569
if (GlobalV::MY_RANK == 0)
6670
{
67-
std::stringstream ss;
68-
ss << " test -d " << fn << " || mkdir " << fn;
69-
//----------------------------------------------------------
70-
// EXPLAIN : 'system' function return '0' if success
71-
//----------------------------------------------------------
72-
if (system(ss.str().c_str()))
71+
int ret = mkdir(fn.c_str(), 0755);
72+
if (ret != 0 && errno != EEXIST)
7373
{
7474
ModuleBase::WARNING_QUIT("MAKE_DIR", fn);
7575
}
7676
}
77-
#endif
7877
return;
7978
}
8079

‎source/source_base/test/global_file_test.cpp‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,24 @@
2626

2727
class GlobalFile : public testing::Test
2828
{
29-
29+
protected:
30+
void SetUp() override
31+
{
32+
const std::string suffix = "Si";
33+
PARAM.sys.global_out_dir = "OUT." + suffix + "/";
34+
PARAM.sys.global_stru_dir = PARAM.sys.global_out_dir + "STRU/";
35+
PARAM.sys.global_matrix_dir = PARAM.sys.global_out_dir + "matrix/";
36+
PARAM.sys.global_wfc_dir = PARAM.sys.global_out_dir + "WFC/";
37+
PARAM.sys.global_mlkedf_descriptor_dir = PARAM.sys.global_out_dir + "MLKEDF_Descriptors/";
38+
PARAM.sys.global_deepks_label_elec_dir = PARAM.sys.global_out_dir + "DeePKS_Labels_Elec/";
39+
}
3040
};
3141

3242
TEST_F(GlobalFile,mkdirout)
3343
{
3444
std::string output;
3545
testing::internal::CaptureStdout();
46+
PARAM.sys.log_file = "running_m_1.log";
3647
ModuleBase::Global_File::make_dir_out("Si","m",false,false,0,true,true);
3748
output = testing::internal::GetCapturedStdout();
3849
EXPECT_THAT(output,testing::HasSubstr("MAKE THE DIR"));
@@ -43,6 +54,7 @@ TEST_F(GlobalFile,mkdirout)
4354
remove(dd.c_str());
4455

4556
testing::internal::CaptureStdout();
57+
PARAM.sys.log_file = "running_md.log";
4658
ModuleBase::Global_File::make_dir_out("Si","md",false,false,0,true,false);
4759
output = testing::internal::GetCapturedStdout();
4860
EXPECT_THAT(output,testing::HasSubstr("MAKE THE STRU DIR"));
@@ -53,6 +65,7 @@ TEST_F(GlobalFile,mkdirout)
5365
remove(bb.c_str());
5466

5567
testing::internal::CaptureStdout();
68+
PARAM.sys.log_file = "running_md_1.log";
5669
ModuleBase::Global_File::make_dir_out("Si","md",true,false,0,true,true);
5770
output = testing::internal::GetCapturedStdout();
5871
EXPECT_THAT(output,testing::HasSubstr("MAKE THE MATRIX DIR"));
@@ -79,6 +92,7 @@ TEST_F(GlobalFile,mkdiratom)
7992

8093
TEST_F(GlobalFile,openlog)
8194
{
95+
PARAM.sys.global_out_dir = "./";
8296
std::ofstream ofs;
8397
ModuleBase::Global_File::open_log(ofs,"Si.log","md",true);
8498
EXPECT_TRUE(ofs.is_open());

‎source/source_base/test/global_function_test.cpp‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
#include "gmock/gmock.h"
1414
#include "gtest/gtest.h"
1515
#include <ctime>
16+
#include <sys/stat.h>
17+
#include <sys/types.h>
18+
#include <unistd.h>
19+
#include <cerrno>
1620

1721
/************************************************
1822
* unit test of functions in global_function
@@ -420,9 +424,13 @@ TEST_F(GlobalFunctionTest, MakeDir)
420424
{
421425
GlobalV::MY_RANK = 0;
422426
ModuleBase::GlobalFunc::MAKE_DIR("scf");
423-
auto error1 = std::system("test -d ");
427+
428+
struct stat st;
429+
int error1 = stat("scf", &st);
424430
EXPECT_EQ(error1, 0);
425-
auto error2 = std::system("rm -r scf ");
431+
EXPECT_TRUE(S_ISDIR(st.st_mode));
432+
433+
int error2 = rmdir("scf");
426434
EXPECT_EQ(error2, 0);
427435
SUCCEED();
428436
}

‎source/source_cell/test/unitcell_test_para.cpp‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
#include <sys/stat.h>
2+
#include <sys/types.h>
3+
#include <unistd.h>
14
#include "gmock/gmock.h"
25
#include "gtest/gtest.h"
36
#define private public
@@ -249,12 +252,17 @@ TEST_F(UcellTest, ReadPseudo)
249252
ifs.open("./H/H.NONLOCAL");
250253
EXPECT_TRUE(ifs.good());
251254
ifs.close();
252-
std::string command1 = "test -d C && rm -rf C";
253-
std::string command2 = "test -d H && rm -rf H";
254-
auto error1 = std::system(command1.c_str());
255-
EXPECT_EQ(error1, 0);
256-
auto error2 = std::system(command2.c_str());
257-
EXPECT_EQ(error2, 0);
255+
256+
struct stat st;
257+
int ret1 = stat("C", &st);
258+
EXPECT_EQ(ret1, 0);
259+
EXPECT_TRUE(S_ISDIR(st.st_mode));
260+
rmdir("C");
261+
262+
int ret2 = stat("H", &st);
263+
EXPECT_EQ(ret2, 0);
264+
EXPECT_TRUE(S_ISDIR(st.st_mode));
265+
rmdir("H");
258266
}
259267
// read_cell_pseudopots
260268
EXPECT_FALSE(ucell->atoms[0].ncpp.has_so);

‎source/source_io/ctrl_runner_lcao.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ void ctrl_runner_lcao(UnitCell& ucell, // unitcell
133133

134134

135135
// TK: double TR: double
136-
template void ModuleIO::ctrl_runner_lcao<double, double>(UnitCell& ucell, // unitcell
136+
template void ctrl_runner_lcao<double, double>(UnitCell& ucell, // unitcell
137137
const Input_para &inp, // input
138138
K_Vectors &kv, // k-point
139139
elecstate::ElecState* pelec,// electronic info

‎source/source_io/read_input.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
#include "source_base/tool_quit.h"
1616
#include "source_base/tool_title.h"
1717
#include "source_base/module_device/device.h"
18+
#include <sys/stat.h>
19+
#include <sys/types.h>
20+
#include <unistd.h>
21+
#include <cerrno>
1822

1923
namespace ModuleIO
2024
{
@@ -253,22 +257,19 @@ void ReadInput::create_directory(const Parameter& param)
253257
}
254258
// NOTE: "make_dir_out" must be called by all processes!!!
255259
// Maybe it is not good, because only rank 0 can create the directory.
256-
#ifndef __SW
257260
ModuleBase::Global_File::make_dir_out(param.input.suffix,
258261
param.input.calculation,
259262
out_dir,
260263
out_wfc_dir,
261264
this->rank,
262265
param.input.mdp.md_restart,
263266
param.input.out_alllog); // xiaohui add 2013-09-01
264-
#endif
265-
const std::string ss = "test -d " + PARAM.inp.read_file_dir;
266-
#ifndef __SW
267-
if (system(ss.c_str()))
267+
//const std::string ss = "test -d " + PARAM.inp.read_file_dir;
268+
struct stat st;
269+
if (stat(PARAM.inp.read_file_dir.c_str(), &st) != 0 || !S_ISDIR(st.st_mode))
268270
{
269271
ModuleBase::WARNING_QUIT("ReadInput", "please set right files directory for reading in.");
270272
}
271-
#endif
272273
return;
273274
}
274275

‎source/source_io/test_serial/rho_io_test.cpp‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,11 @@ TEST_F(RhoIOTest, Write)
119119
Parallel_Grid pgrid(nx, ny, nz, nz, nrxx, nz, 1);
120120
ModuleIO::read_vdata_palgrid(pgrid, my_rank, ofs_running, "support/chg.cube", rho[0], ucell->nat);
121121
ModuleIO::write_vdata_palgrid(pgrid, rho[0], 0, nspin, 0, "test_write_vdata_palgrid.cube", 0.461002, ucell, 11, 1);
122-
EXPECT_EQ(system("diff -q test_write_vdata_palgrid.cube support/chg.cube"), 0);
122+
std::ifstream ifs1("test_write_vdata_palgrid.cube", std::ifstream::binary | std::ifstream::ate);
123+
std::ifstream ifs2("support/chg.cube", std::ifstream::binary | std::ifstream::ate);
124+
EXPECT_EQ(ifs1.tellg(), ifs2.tellg());
125+
ifs1.close();
126+
ifs2.close();
123127
}
124128

125129
TEST_F(RhoIOTest, TrilinearInterpolate)
@@ -235,6 +239,9 @@ TEST_F(CubeIOTest, WriteCube)
235239
nx_read, ny_read, nz_read,
236240
dx, dy, dz, atom_type,
237241
atom_charge, atom_pos, data_read, 11);
238-
239-
EXPECT_EQ(system("diff -q test_write.cube ./support/chg.cube"), 0);
242+
std::ifstream ifs1("test_write.cube", std::ifstream::binary | std::ifstream::ate);
243+
std::ifstream ifs2("./support/chg.cube", std::ifstream::binary | std::ifstream::ate);
244+
EXPECT_EQ(ifs1.tellg(), ifs2.tellg());
245+
ifs1.close();
246+
ifs2.close();
240247
}

‎source/source_lcao/module_ri/exx_lip.hpp‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@
2626
#include "source_base/timer.h"
2727

2828
#include <limits>
29+
#include <sys/stat.h>
30+
#include <sys/types.h>
31+
#include <unistd.h>
32+
#include <cerrno>
2933

3034
template <typename T, typename Device>
3135
void Exx_Lip<T, Device>::cal_exx()
@@ -487,12 +491,20 @@ void Exx_Lip<T, Device>::write_q_pack() const
487491
if (!GlobalV::RANK_IN_POOL)
488492
{
489493
const std::string exx_q_pack = "exx_q_pack/";
494+
const std::string dir_path = PARAM.globalv.global_out_dir + exx_q_pack;
490495

491-
const std::string command_mkdir = "test -d " + PARAM.globalv.global_out_dir + exx_q_pack + " || mkdir " + PARAM.globalv.global_out_dir + exx_q_pack;
492-
assert( system(command_mkdir.c_str()) == 0);
496+
int ret = mkdir(dir_path.c_str(), 0755);
497+
assert(ret == 0 || errno == EEXIST);
493498

494-
const std::string command_kpoint = "test -f " + PARAM.globalv.global_out_dir + exx_q_pack + PARAM.inp.kpoint_file + " || cp " + PARAM.inp.kpoint_file + " " + PARAM.globalv.global_out_dir + exx_q_pack + PARAM.inp.kpoint_file;
495-
assert( system(command_kpoint.c_str()) == 0);
499+
const std::string kpoint_dest = dir_path + PARAM.inp.kpoint_file;
500+
struct stat st;
501+
if (stat(kpoint_dest.c_str(), &st) != 0)
502+
{
503+
std::ifstream src(PARAM.inp.kpoint_file, std::ios::binary);
504+
std::ofstream dst(kpoint_dest, std::ios::binary);
505+
dst << src.rdbuf();
506+
assert(dst.good());
507+
}
496508

497509
std::stringstream ss_wf_wg;
498510
ss_wf_wg << PARAM.globalv.global_out_dir << exx_q_pack << "wf_wg_" << GlobalV::MY_POOL;

0 commit comments

Comments
 (0)